From 9da7ea1bbc8ae4b90cb92bdf228cdcf9dd6f18f4 Mon Sep 17 00:00:00 2001 From: Evan Buss Date: Sun, 11 Aug 2024 18:47:46 +0000 Subject: [PATCH] feat: environment configuration Environment variables can now be used to configure any config property including the feeds list. This makes it easier to use in environments without access to config files like GCR. Some may prefer not to have a separate config file as well. Also added build metadata to the docker image and binaries. --- .air.toml | 2 +- .devcontainer/setup.sh | 3 +- .github/workflows/action.yml | 4 ++ README.md | 10 +++ dockerfile | 6 +- go.mod | 15 +++- go.sum | 16 ++++- internal/envextended/provider.go | 105 ++++++++++++++++++++++++++++ main.go | 115 ++++++++++++++++++++++--------- server.go | 2 +- 10 files changed, 239 insertions(+), 39 deletions(-) create mode 100644 internal/envextended/provider.go diff --git a/.air.toml b/.air.toml index 75ecbe1..2193576 100644 --- a/.air.toml +++ b/.air.toml @@ -3,7 +3,7 @@ testdata_dir = "testdata" tmp_dir = "tmp" [build] - args_bin = ["--dev"] + args_bin = [] bin = "./tmp/main" cmd = "go build -o ./tmp/main ." delay = 1000 diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index 02fbaf3..68968c6 100755 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -13,4 +13,5 @@ cp kindlegen/kindlegen /usr/local/bin/kindlegen chmod +x /usr/local/bin/kindlegen rm -rf kindlegen kindlegen_linux_2.6_i386_v2_9.tar.gz -go install github.com/air-verse/air@latest \ No newline at end of file +go install github.com/air-verse/air@latest +go install github.com/goreleaser/goreleaser/v2@latest \ No newline at end of file diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index b86919e..28de460 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -71,6 +71,10 @@ jobs: platforms: ${{ steps.set-platforms.outputs.DOCKER_PLATFORMS}} cache-from: type=gha cache-to: type=gha,mode=max + build-args: + VERSION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }} + REVISION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }} + BUILDTIME=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }} - name: Docker Hub Description uses: peter-evans/dockerhub-description@e98e4d1628a5f3be2be7c231e50981aee98723ae # v4.0.0 diff --git a/README.md b/README.md index a8b15aa..b66e968 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ By running your own OPDS Proxy you can allow eReaders to navigate and download b You can run OPDS Proxy as a Docker container or as an executable. +Configuration is done via YAML file (default `config.yml`), environment variables, and command flags in that order of precedence. + ### Docker Docker images are published to [Docker Hub](https://hub.docker.com/r/evanbuss/opds-proxy) and the [GitHub Container Registry](https://github.com/evan-buss/opds-proxy/pkgs/container/opds-proxy). @@ -32,6 +34,14 @@ services: image: evanbuss/opds-proxy:latest #image: ghcr.io/evan-buss/opds-proxy:latest container_name: opds-proxy + # You can also use environment variables to configure the container + # environment: + # - OPDS__PORT=5228 + # - OPDS__FEEDS__0__NAME=Some Feed + # - OPDS__FEEDS__0__URL=http://some-feed.com/opds + # - OPDS__FEEDS__0__AUTH__USERNAME=user + # - OPDS__FEEDS__0__AUTH__PASSWORD=password + # - OPDS__FEEDS__0__AUTH__LOCAL_ONLY=true ports: - 8080:8080 volumes: diff --git a/dockerfile b/dockerfile index 9255daf..c059d8d 100644 --- a/dockerfile +++ b/dockerfile @@ -23,7 +23,11 @@ RUN go mod verify COPY . . -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -w" -o opds-proxy +ARG VERSION=dev +ARG REVISION=unknown +ARG BUILDTIME=unknown + +RUN CGO_ENABLED=0 go build -ldflags "-s -w -X main.version=${VERSION} -X main.commit=${REVISION} -X main.date=${BUILDTIME}" -o opds-proxy FROM gcr.io/distroless/static diff --git a/go.mod b/go.mod index e49c126..6f2c7d1 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,16 @@ require ( github.com/knadh/koanf/v2 v2.1.1 ) -require github.com/gorilla/securecookie v1.1.2 +require ( + github.com/gorilla/securecookie v1.1.2 + github.com/spf13/pflag v1.0.5 +) + +require ( + github.com/tidwall/gjson v1.14.2 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect +) require ( github.com/fsnotify/fsnotify v1.7.0 // indirect @@ -16,9 +25,11 @@ require ( github.com/go-viper/mapstructure/v2 v2.0.0 // indirect github.com/google/uuid v1.6.0 github.com/knadh/koanf/maps v0.1.1 // indirect - github.com/knadh/koanf/providers/confmap v0.1.0 + github.com/knadh/koanf/parsers/json v0.1.0 + github.com/knadh/koanf/providers/posflag v0.1.0 github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/tidwall/sjson v1.2.5 golang.org/x/sys v0.22.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 3245bbf..a5a2af4 100644 --- a/go.sum +++ b/go.sum @@ -14,12 +14,14 @@ github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kX github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo= github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/parsers/json v0.1.0 h1:dzSZl5pf5bBcW0Acnu20Djleto19T0CfHcvZ14NJ6fU= +github.com/knadh/koanf/parsers/json v0.1.0/go.mod h1:ll2/MlXcZ2BfXD6YJcjVFzhG9P0TdJ207aIBKQhV2hY= github.com/knadh/koanf/parsers/yaml v0.1.0 h1:ZZ8/iGfRLvKSaMEECEBPM1HQslrZADk8fP1XFUxVI5w= github.com/knadh/koanf/parsers/yaml v0.1.0/go.mod h1:cvbUDC7AL23pImuQP0oRw/hPuccrNBS2bps8asS0CwY= -github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= -github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= github.com/knadh/koanf/providers/file v1.0.0 h1:DtPvSQBeF+N0QLPMz0yf2bx0nFSxUcncpqQvzCxfCyk= github.com/knadh/koanf/providers/file v1.0.0/go.mod h1:/faSBcv2mxPVjFrXck95qeoyoZ5myJ6uxN8OOVNJJCI= +github.com/knadh/koanf/providers/posflag v0.1.0 h1:mKJlLrKPcAP7Ootf4pBZWJ6J+4wHYujwipe7Ie3qW6U= +github.com/knadh/koanf/providers/posflag v0.1.0/go.mod h1:SYg03v/t8ISBNrMBRMlojH8OsKowbkXV7giIbBVgbz0= github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM= github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= @@ -28,8 +30,18 @@ github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zx github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tidwall/gjson v1.14.2 h1:6BBkirS0rAHjumnjHF6qgy5d2YAJ1TLIaFE2lzfOLqo= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/internal/envextended/provider.go b/internal/envextended/provider.go new file mode 100644 index 0000000..6e54401 --- /dev/null +++ b/internal/envextended/provider.go @@ -0,0 +1,105 @@ +package envextended + +import ( + "errors" + "os" + "strings" + + "github.com/tidwall/sjson" +) + +type Env struct { + prefix string + delim string + cb func(key string, value string) (string, interface{}) + out string +} + +func Provider(prefix, delim string, cb func(s string) string) *Env { + e := &Env{ + prefix: prefix, + delim: delim, + out: "{}", + } + if cb != nil { + e.cb = func(key string, value string) (string, interface{}) { + return cb(key), value + } + } + return e +} + +// ProviderWithValue works exactly the same as Provider except the callback +// takes a (key, value) with the variable name and value and allows you +// to modify both. This is useful for cases where you may want to return +// other types like a string slice instead of just a string. +func ProviderWithValue(prefix, delim string, cb func(key string, value string) (string, interface{})) *Env { + return &Env{ + prefix: prefix, + delim: delim, + cb: cb, + } +} + +// ReadBytes reads the contents of a file on disk and returns the bytes. +func (e *Env) ReadBytes() ([]byte, error) { + // Collect the environment variable keys. + var keys []string + for _, k := range os.Environ() { + if e.prefix != "" { + if strings.HasPrefix(k, e.prefix) { + keys = append(keys, k) + } + } else { + keys = append(keys, k) + } + } + + for _, k := range keys { + parts := strings.SplitN(k, "=", 2) + + var ( + key string + value interface{} + ) + + // If there's a transformation callback, + // run it through every key/value. + if e.cb != nil { + key, value = e.cb(parts[0], parts[1]) + // If the callback blanked the key, it should be omitted + if key == "" { + continue + } + } else { + key = parts[0] + value = parts[1] + } + + if err := e.set(key, value); err != nil { + return []byte{}, err + } + } + + if e.out == "" { + return []byte("{}"), nil + } + + return []byte(e.out), nil +} + +func (e *Env) set(key string, value interface{}) error { + out, err := sjson.Set(e.out, strings.Replace(key, e.delim, ".", -1), value) + if err != nil { + return err + } + + e.out = out + + return nil +} + +// Read is not supported by the file provider. +func (e *Env) Read() (map[string]interface{}, error) { + return nil, errors.New("envextended provider does not support this method") +} diff --git a/main.go b/main.go index 328c2aa..76ad672 100644 --- a/main.go +++ b/main.go @@ -2,24 +2,33 @@ package main import ( "encoding/hex" - "flag" + "errors" "fmt" "log" "net/http" "os" + "strings" + "github.com/evan-buss/opds-proxy/internal/envextended" "github.com/gorilla/securecookie" + "github.com/knadh/koanf/parsers/json" "github.com/knadh/koanf/parsers/yaml" - "github.com/knadh/koanf/providers/confmap" "github.com/knadh/koanf/providers/file" + "github.com/knadh/koanf/providers/posflag" "github.com/knadh/koanf/v2" + flag "github.com/spf13/pflag" ) +// Version information set at build time +var version = "dev" +var commit = "unknown" +var date = "unknown" + type ProxyConfig struct { Port string `koanf:"port"` Auth AuthConfig `koanf:"auth"` Feeds []FeedConfig `koanf:"feeds" ` - isDevMode bool + DebugMode bool `koanf:"debug"` } type AuthConfig struct { @@ -40,44 +49,54 @@ type FeedConfigAuth struct { } func main() { - fs := flag.NewFlagSet("", flag.ContinueOnError) - // These aren't mapped to the config file. - configPath := fs.String("config", "config.yml", "config file to load") - generateKeys := fs.Bool("generate-keys", false, "generate cookie signing keys and exit") - isDevMode := fs.Bool("dev", false, "enable development mode") + var k = koanf.New(".") - port := fs.String("port", "8080", "port to listen on") + fs := flag.NewFlagSet("", flag.ContinueOnError) + fs.StringP("port", "p", "8080", "port to listen on") + fs.StringP("config", "c", "config.yml", "config file to load") + fs.Bool("generate-keys", false, "generate cookie signing keys and exit") + fs.BoolP("version", "v", false, "print version and exit") + fs.Usage = func() { + fmt.Println("Usage: opds-proxy [flags]") + fmt.Println(fs.FlagUsages()) + os.Exit(0) + } if err := fs.Parse(os.Args[1:]); err != nil { - log.Fatal(err) + log.Fatalf("error parsing flags: %v", err) } - if *generateKeys { + if showVersion, _ := fs.GetBool("version"); showVersion { + fmt.Println("opds-proxy") + fmt.Printf(" Version: %s\n", version) + fmt.Printf(" Commit: %s\n", commit) + fmt.Printf(" Build Date: %s\n", date) + os.Exit(0) + } + + if generate, _ := fs.GetBool("generate-keys"); generate { displayKeys() os.Exit(0) } - var k = koanf.New(".") - - // Load config file from disk. - // Feed options must be defined here. - if err := k.Load(file.Provider(*configPath), yaml.Parser()); err != nil && !os.IsNotExist(err) { - log.Fatal(err) + // YAML Config + configPath, _ := fs.GetString("config") + if err := k.Load(file.Provider(configPath), yaml.Parser()); err != nil && !os.IsNotExist(err) { + log.Fatalf("error loading config file: %v", err) } - // Selectively add command line options to the config. Overriding the config file. - if err := k.Load(confmap.Provider(map[string]interface{}{ - "port": *port, - }, "."), nil); err != nil { - log.Fatal(err) + // Environment Variables Config + if err := k.Load(envextended.ProviderWithValue("OPDS", ".", envCallback), json.Parser()); err != nil { + log.Fatalf("error loading environment variables: %v", err) + } + + // CLI Flags Config + if err := k.Load(posflag.Provider(fs, ".", k), nil); err != nil { + log.Fatalf("error loading CLI flags: %v", err) } config := ProxyConfig{} k.Unmarshal("", &config) - if len(config.Feeds) == 0 { - log.Fatal("No feeds defined in config") - } - if config.Auth.HashKey == "" || config.Auth.BlockKey == "" { log.Println("Generating new cookie signing credentials") hashKey, blockKey := displayKeys() @@ -86,16 +105,17 @@ func main() { config.Auth.BlockKey = blockKey } - // This should only be set by the command line flag, - // so we don't use koanf to set this. - config.isDevMode = *isDevMode + if err := config.Validate(); err != nil { + log.Fatalf("invalid configuration: %v", err) + } + server, err := NewServer(&config) if err != nil { - log.Fatal(err) + log.Fatalf("error creating server: %v", err) } if err = server.Serve(); err != nil && err != http.ErrServerClosed { - log.Fatal(err) + log.Fatalf("error serving: %v", err) } } @@ -110,3 +130,36 @@ func displayKeys() (string, string) { return hashKey, blockKey } + +func envCallback(key string, value string) (string, interface{}) { + key = strings.TrimPrefix(key, "OPDS__") + key = strings.ReplaceAll(key, "__", ".") + key = strings.ToLower(key) + return key, value +} + +func (c *ProxyConfig) Validate() error { + if c.Port == "" { + return errors.New("port is required") + } + + if c.Auth.HashKey == "" || c.Auth.BlockKey == "" { + return errors.New("auth.hash_key and auth.block_key are required") + } + + if len(c.Feeds) == 0 { + return errors.New("at least one feed must be defined") + } + + for _, feed := range c.Feeds { + if feed.Name == "" { + return errors.New("feed.name is required") + } + + if feed.Url == "" { + return errors.New("feed.url is required") + } + } + + return nil +} diff --git a/server.go b/server.go index 7a18375..caf278d 100644 --- a/server.go +++ b/server.go @@ -66,7 +66,7 @@ func NewServer(config *ProxyConfig) (*Server, error) { return nil, err } - if !config.isDevMode { + if !config.DebugMode { slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil))) }