diff --git a/cmd/frontend/main.go b/cmd/frontend/main.go index f931d85ac..2a05e14c4 100644 --- a/cmd/frontend/main.go +++ b/cmd/frontend/main.go @@ -18,6 +18,7 @@ import ( "golang.org/x/pkgsite/cmd/internal/cmdconfig" "golang.org/x/pkgsite/internal" "golang.org/x/pkgsite/internal/config" + "golang.org/x/pkgsite/internal/config/serverconfig" "golang.org/x/pkgsite/internal/dcensus" "golang.org/x/pkgsite/internal/fetch" "golang.org/x/pkgsite/internal/fetchdatasource" @@ -35,7 +36,7 @@ import ( ) var ( - queueName = config.GetEnv("GO_DISCOVERY_FRONTEND_TASK_QUEUE", "") + queueName = serverconfig.GetEnv("GO_DISCOVERY_FRONTEND_TASK_QUEUE", "") workers = flag.Int("workers", 10, "number of concurrent requests to the fetch service, when running locally") staticFlag = flag.String("static", "static", "path to folder containing static files served") thirdPartyPath = flag.String("third_party", "third_party", "path to folder containing third-party libraries") @@ -53,7 +54,7 @@ var ( func main() { flag.Parse() ctx := context.Background() - cfg, err := config.Init(ctx) + cfg, err := serverconfig.Init(ctx) if err != nil { log.Fatal(ctx, err) } @@ -169,7 +170,7 @@ func main() { } // We are not currently forwarding any ports on AppEngine, so serving debug // information is broken. - if !cfg.OnAppEngine() { + if !serverconfig.OnAppEngine() { dcensusServer, err := dcensus.NewServer() if err != nil { log.Fatal(ctx, err) diff --git a/cmd/internal/cmdconfig/cmdconfig.go b/cmd/internal/cmdconfig/cmdconfig.go index 3b5d4d5ee..6d46e0948 100644 --- a/cmd/internal/cmdconfig/cmdconfig.go +++ b/cmd/internal/cmdconfig/cmdconfig.go @@ -19,6 +19,7 @@ import ( "golang.org/x/pkgsite/internal" "golang.org/x/pkgsite/internal/config" "golang.org/x/pkgsite/internal/config/dynconfig" + "golang.org/x/pkgsite/internal/config/serverconfig" "golang.org/x/pkgsite/internal/database" "golang.org/x/pkgsite/internal/derrors" "golang.org/x/pkgsite/internal/log" @@ -30,12 +31,12 @@ import ( // Logger configures a middleware.Logger. func Logger(ctx context.Context, cfg *config.Config, logName string) middleware.Logger { - if cfg.OnGCP() { + if serverconfig.OnGCP() { opts := []logging.LoggerOption{logging.CommonResource(&mrpb.MonitoredResource{ Type: cfg.MonitoredResource.Type, Labels: cfg.MonitoredResource.Labels, })} - if cfg.OnGKE() { + if serverconfig.OnGKE() { opts = append(opts, logging.CommonLabels(map[string]string{ "k8s-pod/env": cfg.DeploymentEnvironment(), "k8s-pod/app": cfg.Application(), @@ -53,7 +54,7 @@ func Logger(ctx context.Context, cfg *config.Config, logName string) middleware. // Reporter configures an Error Reporting client. func Reporter(ctx context.Context, cfg *config.Config) derrors.Reporter { - if !cfg.OnGCP() || cfg.DisableErrorReporting { + if !serverconfig.OnGCP() || cfg.DisableErrorReporting { return nil } reportingClient, err := errorreporting.NewClient(ctx, cfg.ProjectID, errorreporting.Config{ diff --git a/cmd/worker/main.go b/cmd/worker/main.go index d500159c2..495d1b37a 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -19,6 +19,7 @@ import ( _ "github.com/jackc/pgx/v4/stdlib" // for pgx driver "golang.org/x/pkgsite/cmd/internal/cmdconfig" "golang.org/x/pkgsite/internal/config" + "golang.org/x/pkgsite/internal/config/serverconfig" "golang.org/x/pkgsite/internal/dcensus" "golang.org/x/pkgsite/internal/index" "golang.org/x/pkgsite/internal/log" @@ -31,8 +32,8 @@ import ( ) var ( - timeout = config.GetEnvInt(context.Background(), "GO_DISCOVERY_WORKER_TIMEOUT_MINUTES", 10) - queueName = config.GetEnv("GO_DISCOVERY_WORKER_TASK_QUEUE", "") + timeout = serverconfig.GetEnvInt(context.Background(), "GO_DISCOVERY_WORKER_TIMEOUT_MINUTES", 10) + queueName = serverconfig.GetEnv("GO_DISCOVERY_WORKER_TASK_QUEUE", "") workers = flag.Int("workers", 10, "number of concurrent requests to the fetch service, when running locally") // flag used in call to safehtml/template.TrustedSourceFromFlag _ = flag.String("static", "static", "path to folder containing static files served") @@ -44,7 +45,7 @@ func main() { ctx := context.Background() - cfg, err := config.Init(ctx) + cfg, err := serverconfig.Init(ctx) if err != nil { log.Fatal(ctx, err) } @@ -128,7 +129,7 @@ func main() { } // We are not currently forwarding any ports on AppEngine, so serving debug // information is broken. - if !cfg.OnAppEngine() { + if !serverconfig.OnAppEngine() { dcensusServer, err := dcensus.NewServer() if err != nil { log.Fatal(ctx, err) diff --git a/devtools/cmd/db/main.go b/devtools/cmd/db/main.go index 848a4915e..d2e5a03a7 100644 --- a/devtools/cmd/db/main.go +++ b/devtools/cmd/db/main.go @@ -14,7 +14,7 @@ import ( "strings" _ "github.com/jackc/pgx/v4/stdlib" // for pgx driver - "golang.org/x/pkgsite/internal/config" + "golang.org/x/pkgsite/internal/config/serverconfig" "golang.org/x/pkgsite/internal/database" "golang.org/x/pkgsite/internal/log" ) @@ -39,12 +39,12 @@ func main() { } ctx := context.Background() - cfg, err := config.Init(ctx) + cfg, err := serverconfig.Init(ctx) if err != nil { log.Fatal(ctx, err) } - dbName := config.GetEnv("GO_DISCOVERY_DATABASE_NAME", "discovery-db") + dbName := serverconfig.GetEnv("GO_DISCOVERY_DATABASE_NAME", "discovery-db") if err := run(ctx, flag.Args()[0], dbName, cfg.DBConnInfo()); err != nil { log.Fatal(ctx, err) } diff --git a/devtools/cmd/seeddb/main.go b/devtools/cmd/seeddb/main.go index d729ca664..5fd7b065a 100644 --- a/devtools/cmd/seeddb/main.go +++ b/devtools/cmd/seeddb/main.go @@ -21,6 +21,7 @@ import ( "golang.org/x/pkgsite/internal" "golang.org/x/pkgsite/internal/config" "golang.org/x/pkgsite/internal/config/dynconfig" + "golang.org/x/pkgsite/internal/config/serverconfig" "golang.org/x/pkgsite/internal/database" "golang.org/x/pkgsite/internal/derrors" "golang.org/x/pkgsite/internal/experiment" @@ -45,7 +46,7 @@ func main() { flag.Parse() ctx := context.Background() - cfg, err := config.Init(ctx) + cfg, err := serverconfig.Init(ctx) if err != nil { log.Fatal(ctx, err) } diff --git a/internal/config/config.go b/internal/config/config.go index 5c07106e6..2b8f04580 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -2,96 +2,21 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package config resolves shared configuration for Go Discovery services, and -// provides functions to access this configuration. -// -// The Init function should be called before using any of the configuration accessors. +// Package config provides the definition of the configuration for the +// frontend. package config import ( - "context" - "encoding/hex" "encoding/json" - "errors" "fmt" "io" - "math/rand" - "net/http" - "os" - "path" - "strconv" "strings" "time" - - "cloud.google.com/go/storage" - "golang.org/x/net/context/ctxhttp" - "golang.org/x/pkgsite/internal/derrors" - "golang.org/x/pkgsite/internal/log" - "golang.org/x/pkgsite/internal/secrets" - "gopkg.in/yaml.v3" ) -// GetEnv looks up the given key from the environment, returning its value if -// it exists, and otherwise returning the given fallback value. -func GetEnv(key, fallback string) string { - if value, ok := os.LookupEnv(key); ok { - return value - } - return fallback -} - -// GetEnvInt looks up the given key from the environment and expects an integer, -// returning the integer value if it exists, and otherwise returning the given -// fallback value. -// If the environment variable has a value but it can't be parsed as an integer, -// GetEnvInt terminates the program. -func GetEnvInt(ctx context.Context, key string, fallback int) int { - if s, ok := os.LookupEnv(key); ok { - v, err := strconv.Atoi(s) - if err != nil { - log.Fatalf(ctx, "bad value %q for %s: %v", s, key, err) - } - return v - } - return fallback -} - -// GetEnvFloat64 looks up the given key from the environment and expects a -// float64, returning the float64 value if it exists, and otherwise returning -// the given fallback value. -func GetEnvFloat64(key string, fallback float64) float64 { - if valueStr, ok := os.LookupEnv(key); ok { - if value, err := strconv.ParseFloat(valueStr, 64); err == nil { - return value - } - } - return fallback -} - // AppVersionFormat is the expected format of the app version timestamp. const AppVersionFormat = "20060102t150405" -// ValidateAppVersion validates that appVersion follows the expected format -// defined by AppVersionFormat. -func ValidateAppVersion(appVersion string) error { - // Accept GKE versions, which start with the docker image name. - if strings.HasPrefix(appVersion, "gcr.io/") { - return nil - } - if _, err := time.Parse(AppVersionFormat, appVersion); err != nil { - // Accept alternative version, used by our AppEngine deployment script. - const altDateFormat = "2006-01-02t15-04" - if len(appVersion) > len(altDateFormat) { - appVersion = appVersion[:len(altDateFormat)] - } - if _, err := time.Parse(altDateFormat, appVersion); err != nil { - return fmt.Errorf("app version %q does not match time formats %q or %q: %v", - appVersion, AppVersionFormat, altDateFormat, err) - } - } - return nil -} - const ( // BypassQuotaAuthHeader is the header key used by the frontend server to know // that a request can bypass the quota server. @@ -215,35 +140,6 @@ func (c *Config) AppVersionLabel() string { return c.FallbackVersionLabel } -// OnAppEngine reports if the current process is running in an AppEngine -// environment. -func (c *Config) OnAppEngine() bool { - return os.Getenv("GAE_ENV") == "standard" -} - -// OnGKE reports whether the current process is running on GKE. -func (c *Config) OnGKE() bool { - return os.Getenv("GO_DISCOVERY_ON_GKE") == "true" -} - -// OnCloudRun reports whether the current process is running on Cloud Run. -func (c *Config) OnCloudRun() bool { - // Use the presence of the environment variables provided by Cloud Run. - // See https://cloud.google.com/run/docs/reference/container-contract. - for _, ev := range []string{"K_SERVICE", "K_REVISION", "K_CONFIGURATION"} { - if os.Getenv(ev) == "" { - return false - } - } - return true -} - -// OnGCP reports whether the current process is running on Google Cloud -// Platform. -func (c *Config) OnGCP() bool { - return c.OnAppEngine() || c.OnGKE() || c.OnCloudRun() -} - // StatementTimeout is the value of the Postgres statement_timeout parameter. // Statements that run longer than this are terminated. // 10 minutes is the App Engine standard request timeout, @@ -341,14 +237,6 @@ func (c *Config) Application() string { } } -// configOverride holds selected config settings that can be dynamically overridden. -type configOverride struct { - DBHost string `yaml:"DBHost"` - DBSecondaryHost string `yaml:"DBSecondaryHost"` - DBName string `yaml:"DBName"` - Quota QuotaSettings `yaml:"Quota"` -} - // QuotaSettings is config for internal/middleware/quota.go type QuotaSettings struct { Enable bool `yaml:"Enable"` @@ -364,229 +252,6 @@ type QuotaSettings struct { HMACKey []byte `json:"-" yaml:"-"` // key for obfuscating IPs } -// Init resolves all configuration values provided by the config package. It -// must be called before any configuration values are used. -func Init(ctx context.Context) (_ *Config, err error) { - defer derrors.Add(&err, "config.Init(ctx)") - // Build a Config from the execution environment, loading some values - // from envvars and others from remote services. - cfg := &Config{ - AuthValues: parseCommaList(os.Getenv("GO_DISCOVERY_AUTH_VALUES")), - IndexURL: GetEnv("GO_MODULE_INDEX_URL", "https://index.golang.org/index"), - ProxyURL: GetEnv("GO_MODULE_PROXY_URL", "https://proxy.golang.org"), - Port: os.Getenv("PORT"), - DebugPort: os.Getenv("DEBUG_PORT"), - // Resolve AppEngine identifiers - ProjectID: os.Getenv("GOOGLE_CLOUD_PROJECT"), - ServiceID: GetEnv("GAE_SERVICE", os.Getenv("GO_DISCOVERY_SERVICE")), - // Version ID from either AppEngine, Cloud Run (see - // https://cloud.google.com/run/docs/reference/container-contract) or - // GKE (set by our own config). - VersionID: GetEnv("GAE_VERSION", GetEnv("K_REVISION", os.Getenv("DOCKER_IMAGE"))), - InstanceID: GetEnv("GAE_INSTANCE", os.Getenv("GO_DISCOVERY_INSTANCE")), - GoogleTagManagerID: os.Getenv("GO_DISCOVERY_GOOGLE_TAG_MANAGER_ID"), - QueueURL: os.Getenv("GO_DISCOVERY_QUEUE_URL"), - QueueAudience: os.Getenv("GO_DISCOVERY_QUEUE_AUDIENCE"), - - // LocationID is essentially hard-coded until we figure out a good way to - // determine it programmatically, but we check an environment variable in - // case it needs to be overridden. - LocationID: GetEnv("GO_DISCOVERY_GAE_LOCATION_ID", "us-central1"), - // This fallback should only be used when developing locally. - FallbackVersionLabel: time.Now().Format(AppVersionFormat), - DBHost: chooseOne(GetEnv("GO_DISCOVERY_DATABASE_HOST", "localhost")), - DBUser: GetEnv("GO_DISCOVERY_DATABASE_USER", "postgres"), - DBPassword: os.Getenv("GO_DISCOVERY_DATABASE_PASSWORD"), - DBSecondaryHost: chooseOne(os.Getenv("GO_DISCOVERY_DATABASE_SECONDARY_HOST")), - DBPort: GetEnv("GO_DISCOVERY_DATABASE_PORT", "5432"), - DBName: GetEnv("GO_DISCOVERY_DATABASE_NAME", "discovery-db"), - DBSecret: os.Getenv("GO_DISCOVERY_DATABASE_SECRET"), - DBSSL: GetEnv("GO_DISCOVERY_DATABASE_SSL", "disable"), - RedisCacheHost: os.Getenv("GO_DISCOVERY_REDIS_HOST"), - RedisBetaCacheHost: os.Getenv("GO_DISCOVERY_REDIS_BETA_HOST"), - RedisCachePort: GetEnv("GO_DISCOVERY_REDIS_PORT", "6379"), - Quota: QuotaSettings{ - Enable: os.Getenv("GO_DISCOVERY_ENABLE_QUOTA") == "true", - QPS: GetEnvInt(ctx, "GO_DISCOVERY_QUOTA_QPS", 10), - Burst: 20, // ignored in redis-based quota implementation - MaxEntries: 1000, // ignored in redis-based quota implementation - RecordOnly: func() *bool { - t := (os.Getenv("GO_DISCOVERY_QUOTA_RECORD_ONLY") != "false") - return &t - }(), - AuthValues: parseCommaList(os.Getenv("GO_DISCOVERY_AUTH_VALUES")), - }, - UseProfiler: os.Getenv("GO_DISCOVERY_USE_PROFILER") == "true", - LogLevel: os.Getenv("GO_DISCOVERY_LOG_LEVEL"), - ServeStats: os.Getenv("GO_DISCOVERY_SERVE_STATS") == "true", - DisableErrorReporting: os.Getenv("GO_DISCOVERY_DISABLE_ERROR_REPORTING") == "true", - VulnDB: GetEnv("GO_DISCOVERY_VULN_DB", "https://storage.googleapis.com/go-vulndb"), - } - log.SetLevel(cfg.LogLevel) - - bucket := os.Getenv("GO_DISCOVERY_CONFIG_BUCKET") - config := os.Getenv("GO_DISCOVERY_CONFIG_DYNAMIC") - exclude := os.Getenv("GO_DISCOVERY_EXCLUDED_FILENAME") - if bucket != "" { - if config == "" { - return nil, errors.New("GO_DISCOVERY_CONFIG_DYNAMIC must be set if GO_DISCOVERY_CONFIG_BUCKET is") - } - cfg.DynamicConfigLocation = fmt.Sprintf("gs://%s/%s", bucket, config) - if exclude != "" { - cfg.DynamicExcludeLocation = fmt.Sprintf("gs://%s/%s", bucket, exclude) - } - } else { - cfg.DynamicConfigLocation = config - cfg.DynamicExcludeLocation = exclude - } - if cfg.OnGCP() { - // Zone is not available in the environment but can be queried via the metadata API. - zone, err := gceMetadata(ctx, "instance/zone") - if err != nil { - return nil, err - } - cfg.ZoneID = zone - sa, err := gceMetadata(ctx, "instance/service-accounts/default/email") - if err != nil { - return nil, err - } - cfg.ServiceAccount = sa - switch { - case cfg.OnAppEngine(): - // Use the gae_app monitored resource. It would be better to use the - // gae_instance monitored resource, but that's not currently supported: - // https://cloud.google.com/logging/docs/api/v2/resource-list#resource-types - cfg.MonitoredResource = &MonitoredResource{ - Type: "gae_app", - Labels: map[string]string{ - "project_id": cfg.ProjectID, - "module_id": cfg.ServiceID, - "version_id": cfg.VersionID, - "zone": cfg.ZoneID, - }, - } - case cfg.OnCloudRun(): - cfg.MonitoredResource = &MonitoredResource{ - Type: "cloud_run_revision", - Labels: map[string]string{ - "project_id": cfg.ProjectID, - "service_name": cfg.ServiceID, - "revision_name": cfg.VersionID, - "configuration_name": os.Getenv("K_CONFIGURATION"), - }, - } - case cfg.OnGKE(): - cfg.MonitoredResource = &MonitoredResource{ - Type: "k8s_container", - Labels: map[string]string{ - "project_id": cfg.ProjectID, - "location": path.Base(cfg.ZoneID), - "cluster_name": cfg.DeploymentEnvironment() + "-pkgsite", - "namespace_name": "default", - "pod_name": os.Getenv("HOSTNAME"), - "container_name": cfg.Application(), - }, - } - default: - return nil, errors.New("on GCP but using an unknown product") - } - if cfg.InstanceID == "" { - id, err := gceMetadata(ctx, "instance/id") - if err != nil { - return nil, fmt.Errorf("getting instance ID: %v", err) - } - cfg.InstanceID = id - } - } else { // running locally, perhaps - cfg.MonitoredResource = &MonitoredResource{ - Type: "global", - Labels: map[string]string{"project_id": cfg.ProjectID}, - } - } - if cfg.DBHost == "" { - panic("DBHost is empty; impossible") - } - if cfg.DBSecret != "" { - var err error - cfg.DBPassword, err = secrets.Get(ctx, cfg.DBSecret) - if err != nil { - return nil, fmt.Errorf("could not get database password secret: %v", err) - } - } - if cfg.Quota.Enable { - s, err := secrets.Get(ctx, "quota-hmac-key") - if err != nil { - return nil, err - } - hmacKey, err := hex.DecodeString(s) - if err != nil { - return nil, err - } - if len(hmacKey) < 16 { - return nil, errors.New("HMAC secret must be at least 16 bytes") - } - cfg.Quota.HMACKey = hmacKey - log.Debugf(ctx, "quota enforcement enabled: qps=%d burst=%d maxentry=%d", cfg.Quota.QPS, cfg.Quota.Burst, cfg.Quota.MaxEntries) - } else { - log.Debugf(ctx, "quota enforcement disabled") - } - - // If the -override.yaml file exists in the configured bucket, it - // should provide overrides for selected configuration. - // Use this when you want to fix something in prod quickly, without waiting - // to re-deploy. (Otherwise, do not use it.) - if cfg.DeploymentEnvironment() != "local" { - overrideObj := fmt.Sprintf("%s-override.yaml", cfg.DeploymentEnvironment()) - overrideBytes, err := readOverrideFile(ctx, bucket, overrideObj) - if err != nil { - log.Error(ctx, err) - } else { - log.Infof(ctx, "processing overrides from gs://%s/%s", bucket, overrideObj) - processOverrides(ctx, cfg, overrideBytes) - } - } - return cfg, nil -} - -func readOverrideFile(ctx context.Context, bucketName, objName string) (_ []byte, err error) { - defer derrors.Wrap(&err, "readOverrideFile(ctx, %q)", objName) - - client, err := storage.NewClient(ctx) - if err != nil { - return nil, err - } - defer client.Close() - r, err := client.Bucket(bucketName).Object(objName).NewReader(ctx) - if err != nil { - return nil, err - } - defer r.Close() - return io.ReadAll(r) -} - -func processOverrides(ctx context.Context, cfg *Config, bytes []byte) { - var ov configOverride - if err := yaml.Unmarshal(bytes, &ov); err != nil { - log.Errorf(ctx, "processOverrides: yaml.Unmarshal: %v", err) - return - } - override(ctx, "DBHost", &cfg.DBHost, ov.DBHost) - override(ctx, "DBSecondaryHost", &cfg.DBSecondaryHost, ov.DBSecondaryHost) - override(ctx, "DBName", &cfg.DBName, ov.DBName) - override(ctx, "Quota.QPS", &cfg.Quota.QPS, ov.Quota.QPS) - override(ctx, "Quota.Burst", &cfg.Quota.Burst, ov.Quota.Burst) - override(ctx, "Quota.MaxEntries", &cfg.Quota.MaxEntries, ov.Quota.MaxEntries) - override(ctx, "Quota.RecordOnly", &cfg.Quota.RecordOnly, ov.Quota.RecordOnly) -} - -func override[T comparable](ctx context.Context, name string, field *T, val T) { - var zero T - if val != zero { - *field = val - log.Infof(ctx, "overriding %s with %v", name, val) - } -} - // Dump outputs the current config information to the given Writer. func (c *Config) Dump(w io.Writer) error { fmt.Fprint(w, "config: ") @@ -594,55 +259,3 @@ func (c *Config) Dump(w io.Writer) error { enc.SetIndent("", " ") return enc.Encode(c) } - -// chooseOne selects one entry at random from a whitespace-separated -// string. It returns the empty string if there are no elements. -func chooseOne(configVar string) string { - fields := strings.Fields(configVar) - if len(fields) == 0 { - return "" - } - src := rand.NewSource(time.Now().UnixNano()) - rng := rand.New(src) - return fields[rng.Intn(len(fields))] -} - -// gceMetadata reads a metadata value from GCE. -// For the possible values of name, see -// https://cloud.google.com/appengine/docs/standard/java/accessing-instance-metadata. -func gceMetadata(ctx context.Context, name string) (_ string, err error) { - // See https://cloud.google.com/appengine/docs/standard/java/accessing-instance-metadata. - // (This documentation doesn't exist for Golang, but it seems to work). - defer derrors.Wrap(&err, "gceMetadata(ctx, %q)", name) - - const metadataURL = "http://metadata.google.internal/computeMetadata/v1/" - req, err := http.NewRequest("GET", metadataURL+name, nil) - if err != nil { - return "", fmt.Errorf("http.NewRequest: %v", err) - } - req.Header.Set("Metadata-Flavor", "Google") - resp, err := ctxhttp.Do(ctx, nil, req) - if err != nil { - return "", fmt.Errorf("ctxhttp.Do: %v", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("bad status: %s", resp.Status) - } - bytes, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("io.ReadAll: %v", err) - } - return string(bytes), nil -} - -func parseCommaList(s string) []string { - var a []string - for _, p := range strings.Split(s, ",") { - p = strings.TrimSpace(p) - if p != "" { - a = append(a, p) - } - } - return a -} diff --git a/internal/config/serverconfig/config.go b/internal/config/serverconfig/config.go new file mode 100644 index 000000000..7b37a70d8 --- /dev/null +++ b/internal/config/serverconfig/config.go @@ -0,0 +1,387 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package serverconfig resolves shared configuration for Go Discovery services. +package serverconfig + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "io" + "math/rand" + "net/http" + "os" + "path" + "strconv" + "strings" + "time" + + "cloud.google.com/go/storage" + "golang.org/x/net/context/ctxhttp" + "golang.org/x/pkgsite/internal/config" + "golang.org/x/pkgsite/internal/derrors" + "golang.org/x/pkgsite/internal/log" + "golang.org/x/pkgsite/internal/secrets" + "gopkg.in/yaml.v3" +) + +// GetEnv looks up the given key from the environment, returning its value if +// it exists, and otherwise returning the given fallback value. +func GetEnv(key, fallback string) string { + if value, ok := os.LookupEnv(key); ok { + return value + } + return fallback +} + +// GetEnvInt looks up the given key from the environment and expects an integer, +// returning the integer value if it exists, and otherwise returning the given +// fallback value. +// If the environment variable has a value but it can't be parsed as an integer, +// GetEnvInt terminates the program. +func GetEnvInt(ctx context.Context, key string, fallback int) int { + if s, ok := os.LookupEnv(key); ok { + v, err := strconv.Atoi(s) + if err != nil { + log.Fatalf(ctx, "bad value %q for %s: %v", s, key, err) + } + return v + } + return fallback +} + +// ValidateAppVersion validates that appVersion follows the expected format +// defined by AppVersionFormat. +func ValidateAppVersion(appVersion string) error { + // Accept GKE versions, which start with the docker image name. + if strings.HasPrefix(appVersion, "gcr.io/") { + return nil + } + if _, err := time.Parse(config.AppVersionFormat, appVersion); err != nil { + // Accept alternative version, used by our AppEngine deployment script. + const altDateFormat = "2006-01-02t15-04" + if len(appVersion) > len(altDateFormat) { + appVersion = appVersion[:len(altDateFormat)] + } + if _, err := time.Parse(altDateFormat, appVersion); err != nil { + return fmt.Errorf("app version %q does not match time formats %q or %q: %v", + appVersion, config.AppVersionFormat, altDateFormat, err) + } + } + return nil +} + +// OnAppEngine reports if the current process is running in an AppEngine +// environment. +func OnAppEngine() bool { + return os.Getenv("GAE_ENV") == "standard" +} + +// OnGKE reports whether the current process is running on GKE. +func OnGKE() bool { + return os.Getenv("GO_DISCOVERY_ON_GKE") == "true" +} + +// onCloudRun reports whether the current process is running on Cloud Run. +func onCloudRun() bool { + // Use the presence of the environment variables provided by Cloud Run. + // See https://cloud.google.com/run/docs/reference/container-contract. + for _, ev := range []string{"K_SERVICE", "K_REVISION", "K_CONFIGURATION"} { + if os.Getenv(ev) == "" { + return false + } + } + return true +} + +// OnGCP reports whether the current process is running on Google Cloud +// Platform. +func OnGCP() bool { + return OnAppEngine() || OnGKE() || onCloudRun() +} + +// configOverride holds selected config settings that can be dynamically overridden. +type configOverride struct { + DBHost string `yaml:"DBHost"` + DBSecondaryHost string `yaml:"DBSecondaryHost"` + DBName string `yaml:"DBName"` + Quota config.QuotaSettings `yaml:"Quota"` +} + +// Init resolves all configuration values provided by the config package. It +// must be called before any configuration values are used. +func Init(ctx context.Context) (_ *config.Config, err error) { + defer derrors.Add(&err, "config.Init(ctx)") + // Build a Config from the execution environment, loading some values + // from envvars and others from remote services. + cfg := &config.Config{ + AuthValues: parseCommaList(os.Getenv("GO_DISCOVERY_AUTH_VALUES")), + IndexURL: GetEnv("GO_MODULE_INDEX_URL", "https://index.golang.org/index"), + ProxyURL: GetEnv("GO_MODULE_PROXY_URL", "https://proxy.golang.org"), + Port: os.Getenv("PORT"), + DebugPort: os.Getenv("DEBUG_PORT"), + // Resolve AppEngine identifiers + ProjectID: os.Getenv("GOOGLE_CLOUD_PROJECT"), + ServiceID: GetEnv("GAE_SERVICE", os.Getenv("GO_DISCOVERY_SERVICE")), + // Version ID from either AppEngine, Cloud Run (see + // https://cloud.google.com/run/docs/reference/container-contract) or + // GKE (set by our own config). + VersionID: GetEnv("GAE_VERSION", GetEnv("K_REVISION", os.Getenv("DOCKER_IMAGE"))), + InstanceID: GetEnv("GAE_INSTANCE", os.Getenv("GO_DISCOVERY_INSTANCE")), + GoogleTagManagerID: os.Getenv("GO_DISCOVERY_GOOGLE_TAG_MANAGER_ID"), + QueueURL: os.Getenv("GO_DISCOVERY_QUEUE_URL"), + QueueAudience: os.Getenv("GO_DISCOVERY_QUEUE_AUDIENCE"), + + // LocationID is essentially hard-coded until we figure out a good way to + // determine it programmatically, but we check an environment variable in + // case it needs to be overridden. + LocationID: GetEnv("GO_DISCOVERY_GAE_LOCATION_ID", "us-central1"), + // This fallback should only be used when developing locally. + FallbackVersionLabel: time.Now().Format(config.AppVersionFormat), + DBHost: chooseOne(GetEnv("GO_DISCOVERY_DATABASE_HOST", "localhost")), + DBUser: GetEnv("GO_DISCOVERY_DATABASE_USER", "postgres"), + DBPassword: os.Getenv("GO_DISCOVERY_DATABASE_PASSWORD"), + DBSecondaryHost: chooseOne(os.Getenv("GO_DISCOVERY_DATABASE_SECONDARY_HOST")), + DBPort: GetEnv("GO_DISCOVERY_DATABASE_PORT", "5432"), + DBName: GetEnv("GO_DISCOVERY_DATABASE_NAME", "discovery-db"), + DBSecret: os.Getenv("GO_DISCOVERY_DATABASE_SECRET"), + DBSSL: GetEnv("GO_DISCOVERY_DATABASE_SSL", "disable"), + RedisCacheHost: os.Getenv("GO_DISCOVERY_REDIS_HOST"), + RedisBetaCacheHost: os.Getenv("GO_DISCOVERY_REDIS_BETA_HOST"), + RedisCachePort: GetEnv("GO_DISCOVERY_REDIS_PORT", "6379"), + Quota: config.QuotaSettings{ + Enable: os.Getenv("GO_DISCOVERY_ENABLE_QUOTA") == "true", + QPS: GetEnvInt(ctx, "GO_DISCOVERY_QUOTA_QPS", 10), + Burst: 20, // ignored in redis-based quota implementation + MaxEntries: 1000, // ignored in redis-based quota implementation + RecordOnly: func() *bool { + t := (os.Getenv("GO_DISCOVERY_QUOTA_RECORD_ONLY") != "false") + return &t + }(), + AuthValues: parseCommaList(os.Getenv("GO_DISCOVERY_AUTH_VALUES")), + }, + UseProfiler: os.Getenv("GO_DISCOVERY_USE_PROFILER") == "true", + LogLevel: os.Getenv("GO_DISCOVERY_LOG_LEVEL"), + ServeStats: os.Getenv("GO_DISCOVERY_SERVE_STATS") == "true", + DisableErrorReporting: os.Getenv("GO_DISCOVERY_DISABLE_ERROR_REPORTING") == "true", + VulnDB: GetEnv("GO_DISCOVERY_VULN_DB", "https://storage.googleapis.com/go-vulndb"), + } + log.SetLevel(cfg.LogLevel) + + bucket := os.Getenv("GO_DISCOVERY_CONFIG_BUCKET") + configDynamic := os.Getenv("GO_DISCOVERY_CONFIG_DYNAMIC") + exclude := os.Getenv("GO_DISCOVERY_EXCLUDED_FILENAME") + if bucket != "" { + if configDynamic == "" { + return nil, errors.New("GO_DISCOVERY_CONFIG_DYNAMIC must be set if GO_DISCOVERY_CONFIG_BUCKET is") + } + cfg.DynamicConfigLocation = fmt.Sprintf("gs://%s/%s", bucket, configDynamic) + if exclude != "" { + cfg.DynamicExcludeLocation = fmt.Sprintf("gs://%s/%s", bucket, exclude) + } + } else { + cfg.DynamicConfigLocation = configDynamic + cfg.DynamicExcludeLocation = exclude + } + if OnGCP() { + // Zone is not available in the environment but can be queried via the metadata API. + zone, err := gceMetadata(ctx, "instance/zone") + if err != nil { + return nil, err + } + cfg.ZoneID = zone + sa, err := gceMetadata(ctx, "instance/service-accounts/default/email") + if err != nil { + return nil, err + } + cfg.ServiceAccount = sa + switch { + case OnAppEngine(): + // Use the gae_app monitored resource. It would be better to use the + // gae_instance monitored resource, but that's not currently supported: + // https://cloud.google.com/logging/docs/api/v2/resource-list#resource-types + cfg.MonitoredResource = &config.MonitoredResource{ + Type: "gae_app", + Labels: map[string]string{ + "project_id": cfg.ProjectID, + "module_id": cfg.ServiceID, + "version_id": cfg.VersionID, + "zone": cfg.ZoneID, + }, + } + case onCloudRun(): + cfg.MonitoredResource = &config.MonitoredResource{ + Type: "cloud_run_revision", + Labels: map[string]string{ + "project_id": cfg.ProjectID, + "service_name": cfg.ServiceID, + "revision_name": cfg.VersionID, + "configuration_name": os.Getenv("K_CONFIGURATION"), + }, + } + case OnGKE(): + cfg.MonitoredResource = &config.MonitoredResource{ + Type: "k8s_container", + Labels: map[string]string{ + "project_id": cfg.ProjectID, + "location": path.Base(cfg.ZoneID), + "cluster_name": cfg.DeploymentEnvironment() + "-pkgsite", + "namespace_name": "default", + "pod_name": os.Getenv("HOSTNAME"), + "container_name": cfg.Application(), + }, + } + default: + return nil, errors.New("on GCP but using an unknown product") + } + if cfg.InstanceID == "" { + id, err := gceMetadata(ctx, "instance/id") + if err != nil { + return nil, fmt.Errorf("getting instance ID: %v", err) + } + cfg.InstanceID = id + } + } else { // running locally, perhaps + cfg.MonitoredResource = &config.MonitoredResource{ + Type: "global", + Labels: map[string]string{"project_id": cfg.ProjectID}, + } + } + if cfg.DBHost == "" { + panic("DBHost is empty; impossible") + } + if cfg.DBSecret != "" { + var err error + cfg.DBPassword, err = secrets.Get(ctx, cfg.DBSecret) + if err != nil { + return nil, fmt.Errorf("could not get database password secret: %v", err) + } + } + if cfg.Quota.Enable { + s, err := secrets.Get(ctx, "quota-hmac-key") + if err != nil { + return nil, err + } + hmacKey, err := hex.DecodeString(s) + if err != nil { + return nil, err + } + if len(hmacKey) < 16 { + return nil, errors.New("HMAC secret must be at least 16 bytes") + } + cfg.Quota.HMACKey = hmacKey + log.Debugf(ctx, "quota enforcement enabled: qps=%d burst=%d maxentry=%d", cfg.Quota.QPS, cfg.Quota.Burst, cfg.Quota.MaxEntries) + } else { + log.Debugf(ctx, "quota enforcement disabled") + } + + // If the -override.yaml file exists in the configured bucket, it + // should provide overrides for selected configuration. + // Use this when you want to fix something in prod quickly, without waiting + // to re-deploy. (Otherwise, do not use it.) + if cfg.DeploymentEnvironment() != "local" { + overrideObj := fmt.Sprintf("%s-override.yaml", cfg.DeploymentEnvironment()) + overrideBytes, err := readOverrideFile(ctx, bucket, overrideObj) + if err != nil { + log.Error(ctx, err) + } else { + log.Infof(ctx, "processing overrides from gs://%s/%s", bucket, overrideObj) + processOverrides(ctx, cfg, overrideBytes) + } + } + return cfg, nil +} + +func readOverrideFile(ctx context.Context, bucketName, objName string) (_ []byte, err error) { + defer derrors.Wrap(&err, "readOverrideFile(ctx, %q)", objName) + + client, err := storage.NewClient(ctx) + if err != nil { + return nil, err + } + defer client.Close() + r, err := client.Bucket(bucketName).Object(objName).NewReader(ctx) + if err != nil { + return nil, err + } + defer r.Close() + return io.ReadAll(r) +} + +func processOverrides(ctx context.Context, cfg *config.Config, bytes []byte) { + var ov configOverride + if err := yaml.Unmarshal(bytes, &ov); err != nil { + log.Errorf(ctx, "processOverrides: yaml.Unmarshal: %v", err) + return + } + override(ctx, "DBHost", &cfg.DBHost, ov.DBHost) + override(ctx, "DBSecondaryHost", &cfg.DBSecondaryHost, ov.DBSecondaryHost) + override(ctx, "DBName", &cfg.DBName, ov.DBName) + override(ctx, "Quota.QPS", &cfg.Quota.QPS, ov.Quota.QPS) + override(ctx, "Quota.Burst", &cfg.Quota.Burst, ov.Quota.Burst) + override(ctx, "Quota.MaxEntries", &cfg.Quota.MaxEntries, ov.Quota.MaxEntries) + override(ctx, "Quota.RecordOnly", &cfg.Quota.RecordOnly, ov.Quota.RecordOnly) +} + +func override[T comparable](ctx context.Context, name string, field *T, val T) { + var zero T + if val != zero { + *field = val + log.Infof(ctx, "overriding %s with %v", name, val) + } +} + +// chooseOne selects one entry at random from a whitespace-separated +// string. It returns the empty string if there are no elements. +func chooseOne(configVar string) string { + fields := strings.Fields(configVar) + if len(fields) == 0 { + return "" + } + src := rand.NewSource(time.Now().UnixNano()) + rng := rand.New(src) + return fields[rng.Intn(len(fields))] +} + +// gceMetadata reads a metadata value from GCE. +// For the possible values of name, see +// https://cloud.google.com/appengine/docs/standard/java/accessing-instance-metadata. +func gceMetadata(ctx context.Context, name string) (_ string, err error) { + // See https://cloud.google.com/appengine/docs/standard/java/accessing-instance-metadata. + // (This documentation doesn't exist for Golang, but it seems to work). + defer derrors.Wrap(&err, "gceMetadata(ctx, %q)", name) + + const metadataURL = "http://metadata.google.internal/computeMetadata/v1/" + req, err := http.NewRequest("GET", metadataURL+name, nil) + if err != nil { + return "", fmt.Errorf("http.NewRequest: %v", err) + } + req.Header.Set("Metadata-Flavor", "Google") + resp, err := ctxhttp.Do(ctx, nil, req) + if err != nil { + return "", fmt.Errorf("ctxhttp.Do: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("bad status: %s", resp.Status) + } + bytes, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("io.ReadAll: %v", err) + } + return string(bytes), nil +} + +func parseCommaList(s string) []string { + var a []string + for _, p := range strings.Split(s, ",") { + p = strings.TrimSpace(p) + if p != "" { + a = append(a, p) + } + } + return a +} diff --git a/internal/config/config_test.go b/internal/config/serverconfig/config_test.go similarity index 86% rename from internal/config/config_test.go rename to internal/config/serverconfig/config_test.go index dd141fddd..c45f6a0c0 100644 --- a/internal/config/config_test.go +++ b/internal/config/serverconfig/config_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package config +package serverconfig import ( "context" @@ -10,6 +10,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "golang.org/x/pkgsite/internal/config" ) func TestValidateAppVersion(t *testing.T) { @@ -56,10 +57,10 @@ func TestChooseOne(t *testing.T) { func TestProcessOverrides(t *testing.T) { tr := true f := false - cfg := Config{ + cfg := config.Config{ DBHost: "origHost", DBName: "origName", - Quota: QuotaSettings{QPS: 1, Burst: 2, MaxEntries: 3, RecordOnly: &tr}, + Quota: config.QuotaSettings{QPS: 1, Burst: 2, MaxEntries: 3, RecordOnly: &tr}, } ov := ` DBHost: newHost @@ -69,12 +70,12 @@ func TestProcessOverrides(t *testing.T) { ` processOverrides(context.Background(), &cfg, []byte(ov)) got := cfg - want := Config{ + want := config.Config{ DBHost: "newHost", DBName: "origName", - Quota: QuotaSettings{QPS: 1, Burst: 2, MaxEntries: 17, RecordOnly: &f}, + Quota: config.QuotaSettings{QPS: 1, Burst: 2, MaxEntries: 17, RecordOnly: &f}, } - if diff := cmp.Diff(want, got, cmp.AllowUnexported(Config{})); diff != "" { + if diff := cmp.Diff(want, got, cmp.AllowUnexported(config.Config{})); diff != "" { t.Errorf("mismatch (-want, +got):\n%s", diff) } } @@ -108,7 +109,7 @@ func TestEnvAndApp(t *testing.T) { {"-foo-bar", "unknownEnv", "foo-bar"}, {"", "local", "unknownApp"}, } { - cfg := &Config{ServiceID: test.serviceID} + cfg := &config.Config{ServiceID: test.serviceID} gotEnv := cfg.DeploymentEnvironment() if gotEnv != test.wantEnv { t.Errorf("%q: got %q, want %q", test.serviceID, gotEnv, test.wantEnv) diff --git a/internal/database/dbutil.go b/internal/database/dbutil.go index d750fbae9..db6eab649 100644 --- a/internal/database/dbutil.go +++ b/internal/database/dbutil.go @@ -13,7 +13,7 @@ import ( "path/filepath" "strings" - "golang.org/x/pkgsite/internal/config" + "golang.org/x/pkgsite/internal/config/serverconfig" "golang.org/x/pkgsite/internal/derrors" "golang.org/x/pkgsite/internal/testing/testhelper" @@ -31,10 +31,10 @@ import ( // necessary as migrate expects a URI. func DBConnURI(dbName string) string { var ( - user = config.GetEnv("GO_DISCOVERY_DATABASE_USER", "postgres") - password = config.GetEnv("GO_DISCOVERY_DATABASE_PASSWORD", "") - host = config.GetEnv("GO_DISCOVERY_DATABASE_HOST", "localhost") - port = config.GetEnv("GO_DISCOVERY_DATABASE_PORT", "5432") + user = serverconfig.GetEnv("GO_DISCOVERY_DATABASE_USER", "postgres") + password = serverconfig.GetEnv("GO_DISCOVERY_DATABASE_PASSWORD", "") + host = serverconfig.GetEnv("GO_DISCOVERY_DATABASE_HOST", "localhost") + port = serverconfig.GetEnv("GO_DISCOVERY_DATABASE_PORT", "5432") ) cs := fmt.Sprintf("postgres://%s/%s?sslmode=disable&user=%s&password=%s&port=%s&timezone=UTC", host, dbName, url.QueryEscape(user), url.QueryEscape(password), url.QueryEscape(port)) diff --git a/internal/dcensus/dcensus.go b/internal/dcensus/dcensus.go index 3b2f76e47..f67de36a0 100644 --- a/internal/dcensus/dcensus.go +++ b/internal/dcensus/dcensus.go @@ -21,6 +21,7 @@ import ( "go.opencensus.io/trace" "go.opencensus.io/zpages" "golang.org/x/pkgsite/internal/config" + "golang.org/x/pkgsite/internal/config/serverconfig" "golang.org/x/pkgsite/internal/derrors" "golang.org/x/pkgsite/internal/log" ) @@ -169,7 +170,7 @@ func NewViewExporter(cfg *config.Config) (_ *stackdriver.Exporter, err error) { "task_id": cfg.InstanceID, }, } - if cfg.OnGKE() { + if serverconfig.OnGKE() { mr = (*monitoredResource)(cfg.MonitoredResource) } log.Debugf(context.Background(), "monitored resource for monitoring: Type %q, Labels %v", diff --git a/internal/postgres/benchmarks_test.go b/internal/postgres/benchmarks_test.go index 57f2cf253..766e9b029 100644 --- a/internal/postgres/benchmarks_test.go +++ b/internal/postgres/benchmarks_test.go @@ -8,7 +8,7 @@ import ( "context" "testing" - "golang.org/x/pkgsite/internal/config" + "golang.org/x/pkgsite/internal/config/serverconfig" "golang.org/x/pkgsite/internal/database" ) @@ -33,7 +33,7 @@ var testQueries = []string{ func BenchmarkSearch(b *testing.B) { ctx := context.Background() - cfg, err := config.Init(ctx) + cfg, err := serverconfig.Init(ctx) if err != nil { b.Fatal(err) } diff --git a/internal/postgres/requeue.go b/internal/postgres/requeue.go index 618ad0e83..fec80023e 100644 --- a/internal/postgres/requeue.go +++ b/internal/postgres/requeue.go @@ -12,7 +12,7 @@ import ( "strconv" "golang.org/x/pkgsite/internal" - "golang.org/x/pkgsite/internal/config" + "golang.org/x/pkgsite/internal/config/serverconfig" "golang.org/x/pkgsite/internal/derrors" "golang.org/x/pkgsite/internal/log" ) @@ -156,7 +156,7 @@ const largeModulePackageThreshold = 1500 // largeModulesLimit represents the number of large modules that we are // willing to enqueue at a given time. // var for testing. -var largeModulesLimit = config.GetEnvInt(context.Background(), "GO_DISCOVERY_LARGE_MODULES_LIMIT", 100) +var largeModulesLimit = serverconfig.GetEnvInt(context.Background(), "GO_DISCOVERY_LARGE_MODULES_LIMIT", 100) // GetNextModulesToFetch returns the next batch of modules that need to be // processed. We prioritize modules based on (1) whether it has status zero diff --git a/internal/queue/gcpqueue/queue.go b/internal/queue/gcpqueue/queue.go index 23e4ec6ad..bd5685284 100644 --- a/internal/queue/gcpqueue/queue.go +++ b/internal/queue/gcpqueue/queue.go @@ -17,6 +17,7 @@ import ( "time" "cloud.google.com/go/cloudtasks/apiv2" + "golang.org/x/pkgsite/internal/config/serverconfig" taskspb "google.golang.org/genproto/googleapis/cloud/tasks/v2" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -33,7 +34,7 @@ import ( // New creates a new Queue with name queueName based on the configuration // in cfg. When running locally, Queue uses numWorkers concurrent workers. func New(ctx context.Context, cfg *config.Config, queueName string, numWorkers int, expGetter middleware.ExperimentGetter, processFunc queue.InMemoryProcessFunc) (queue.Queue, error) { - if !cfg.OnGCP() { + if !serverconfig.OnGCP() { experiments, err := expGetter(ctx) if err != nil { return nil, err diff --git a/internal/worker/pages.go b/internal/worker/pages.go index 8e6d4b7c0..f35a3260c 100644 --- a/internal/worker/pages.go +++ b/internal/worker/pages.go @@ -19,6 +19,7 @@ import ( "github.com/google/safehtml/template" "golang.org/x/pkgsite/internal" "golang.org/x/pkgsite/internal/config" + "golang.org/x/pkgsite/internal/config/serverconfig" "golang.org/x/pkgsite/internal/derrors" "golang.org/x/pkgsite/internal/log" "golang.org/x/pkgsite/internal/memory" @@ -75,7 +76,7 @@ func (s *Server) doIndexPage(w http.ResponseWriter, r *http.Request) (err error) log.Warningf(ctx, "could not get cgroup stats: %v", err) } var logsURL string - if s.cfg.OnGKE() { + if serverconfig.OnGKE() { env := s.cfg.DeploymentEnvironment() cluster := env + "-" + "pkgsite" logsURL = `https://pantheon.corp.google.com/logs/query;query=resource.type%3D%22k8s_container%22%20resource.labels.cluster_name%3D%22` + diff --git a/internal/worker/server.go b/internal/worker/server.go index be0ac6a1e..a7283b623 100644 --- a/internal/worker/server.go +++ b/internal/worker/server.go @@ -27,6 +27,7 @@ import ( "golang.org/x/pkgsite/internal" "golang.org/x/pkgsite/internal/cache" "golang.org/x/pkgsite/internal/config" + "golang.org/x/pkgsite/internal/config/serverconfig" "golang.org/x/pkgsite/internal/derrors" "golang.org/x/pkgsite/internal/godoc/dochtml" "golang.org/x/pkgsite/internal/index" @@ -605,7 +606,7 @@ func (s *Server) handleReprocess(w http.ResponseWriter, r *http.Request) error { if appVersion == "" { return &serverError{http.StatusBadRequest, errors.New("app_version was not specified")} } - if err := config.ValidateAppVersion(appVersion); err != nil { + if err := serverconfig.ValidateAppVersion(appVersion); err != nil { return &serverError{http.StatusBadRequest, fmt.Errorf("config.ValidateAppVersion(%q): %v", appVersion, err)} } @@ -874,14 +875,14 @@ const mib = 1024 * 1024 var maxModuleZipSize int64 = math.MaxInt64 func init() { - v := config.GetEnvInt(context.Background(), "GO_DISCOVERY_MAX_MODULE_ZIP_MI", -1) + v := serverconfig.GetEnvInt(context.Background(), "GO_DISCOVERY_MAX_MODULE_ZIP_MI", -1) if v > 0 { maxModuleZipSize = int64(v) * mib } } func (s *Server) setLoadShedder(ctx context.Context) { - mebis := config.GetEnvInt(ctx, "GO_DISCOVERY_MAX_IN_FLIGHT_ZIP_MI", -1) + mebis := serverconfig.GetEnvInt(ctx, "GO_DISCOVERY_MAX_IN_FLIGHT_ZIP_MI", -1) if mebis > 0 { log.Infof(ctx, "shedding load over %dMi", mebis) s.loadShedder = &loadShedder{ diff --git a/tests/search/main.go b/tests/search/main.go index 508f4a32f..9e950af92 100755 --- a/tests/search/main.go +++ b/tests/search/main.go @@ -18,7 +18,7 @@ import ( "strings" _ "github.com/jackc/pgx/v4/stdlib" // for pgx driver - "golang.org/x/pkgsite/internal/config" + "golang.org/x/pkgsite/internal/config/serverconfig" "golang.org/x/pkgsite/internal/database" "golang.org/x/pkgsite/internal/derrors" "golang.org/x/pkgsite/internal/frontend" @@ -33,7 +33,7 @@ func main() { flag.Parse() ctx := context.Background() - cfg, err := config.Init(ctx) + cfg, err := serverconfig.Init(ctx) if err != nil { log.Fatal(ctx, err) }