Add regions config option (#1006)

Closes #989.

CHANGELOG.md

@@ -5,6 +5,7 @@ This document outlines major changes between releases.
 ## [Unreleased]
 
 ### Added
+- Config option placement_policy.locations as alternative for placement_policy.region_mapping (#989)
 
 ### Changed
 

cmd/s3-gw/app.go

@@ -190,8 +190,31 @@ func (a *App) initLayer(ctx context.Context, anonSigner user.Signer, neoFS *neof
 	}
 }
 
+func loadLocations(v *viper.Viper) (map[string]string, error) {
+	var (
+		rawLocations = v.GetStringMap(cfgPolicyLocations)
+		locations    = make(map[string]string, len(rawLocations))
+	)
+
+	for key, val := range rawLocations {
+		if s, ok := val.(string); ok {
+			locations[key] = s
+			continue
+		}
+
+		return nil, fmt.Errorf("location %q value is not a string: %s", key, val)
+	}
+
+	return locations, nil
+}
+
 func newAppSettings(log *Logger, v *viper.Viper) *appSettings {
-	policies, err := newPlacementPolicy(getDefaultPolicyValue(v), v.GetString(cfgPolicyRegionMapFile))
+	locations, err := loadLocations(v)
+	if err != nil {
+		log.logger.Fatal("load locations failed", zap.Error(err))
+	}
+
+	policies, err := newPlacementPolicy(getDefaultPolicyValue(v), v.GetString(cfgPolicyRegionMapFile), locations)
 	if err != nil {
 		log.logger.Fatal("failed to create new policy mapping", zap.Error(err))
 	}
@@ -311,12 +334,12 @@ func getPool(ctx context.Context, logger *zap.Logger, cfg *viper.Viper) (*pool.P
 	return p, key, poolStat
 }
 
-func newPlacementPolicy(defaultPolicy string, regionPolicyFilepath string) (*placementPolicy, error) {
+func newPlacementPolicy(defaultPolicy string, regionPolicyFilepath string, locations map[string]string) (*placementPolicy, error) {
 	policies := &placementPolicy{
 		regionMap: make(map[string]netmap.PlacementPolicy),
 	}
 
-	return policies, policies.update(defaultPolicy, regionPolicyFilepath)
+	return policies, policies.update(defaultPolicy, regionPolicyFilepath, locations)
 }
 
 func (p *placementPolicy) Default() netmap.PlacementPolicy {
@@ -333,7 +356,28 @@ func (p *placementPolicy) Get(name string) (netmap.PlacementPolicy, bool) {
 	return policy, ok
 }
 
-func (p *placementPolicy) update(defaultPolicy string, regionPolicyFilepath string) error {
+func parsePolicies(regionMap map[string]netmap.PlacementPolicy, locations map[string]string) error {
+	var err error
+
+	for location, policy := range locations {
+		var pp netmap.PlacementPolicy
+		if err = pp.DecodeString(policy); err == nil {
+			regionMap[location] = pp
+			continue
+		}
+
+		if err = pp.UnmarshalJSON([]byte(policy)); err == nil {
+			regionMap[location] = pp
+			continue
+		}
+
+		return fmt.Errorf("%q: %w", location, err)
+	}
+
+	return nil
+}
+
+func (p *placementPolicy) update(defaultPolicy string, regionPolicyFilepath string, locations map[string]string) error {
 	var defaultPlacementPolicy netmap.PlacementPolicy
 	if err := defaultPlacementPolicy.DecodeString(defaultPolicy); err != nil {
 		return fmt.Errorf("parse default policy '%s': %w", defaultPolicy, err)
@@ -344,20 +388,13 @@ func (p *placementPolicy) update(defaultPolicy string, regionPolicyFilepath stri
 		return fmt.Errorf("read region map file: %w", err)
 	}
 
-	regionMap := make(map[string]netmap.PlacementPolicy, len(regionPolicyMap))
-	for region, policy := range regionPolicyMap {
-		var pp netmap.PlacementPolicy
-		if err = pp.DecodeString(policy); err == nil {
-			regionMap[region] = pp
-			continue
-		}
-
-		if err = pp.UnmarshalJSON([]byte(policy)); err == nil {
-			regionMap[region] = pp
-			continue
-		}
+	regionMap := make(map[string]netmap.PlacementPolicy, len(regionPolicyMap)+len(locations))
+	if err = parsePolicies(regionMap, regionPolicyMap); err != nil {
+		return fmt.Errorf("parse region map: %w", err)
+	}
 
-		return fmt.Errorf("parse region '%s' to policy mapping: %w", region, err)
+	if err = parsePolicies(regionMap, locations); err != nil {
+		return fmt.Errorf("parse locations: %w", err)
 	}
 
 	p.mu.Lock()
@@ -524,7 +561,12 @@ func (a *App) updateSettings() {
 		a.settings.logLevel.SetLevel(lvl)
 	}
 
-	if err := a.settings.policies.update(getDefaultPolicyValue(a.cfg), a.cfg.GetString(cfgPolicyRegionMapFile)); err != nil {
+	regions, err := loadLocations(a.cfg)
+	if err != nil {
+		a.log.Warn("locations won't be updated", zap.Error(err))
+	}
+
+	if err := a.settings.policies.update(getDefaultPolicyValue(a.cfg), a.cfg.GetString(cfgPolicyRegionMapFile), regions); err != nil {
 		a.log.Warn("policies won't be updated", zap.Error(err))
 	}
 }

cmd/s3-gw/app_settings.go

@@ -84,6 +84,7 @@ const ( // Settings.
 	// Policy.
 	cfgPolicyDefault       = "placement_policy.default"
 	cfgPolicyRegionMapFile = "placement_policy.region_mapping"
+	cfgPolicyLocations     = "placement_policy.locations"
 
 	// CORS.
 	cfgDefaultMaxAge = "cors.default_max_age"

config/config.yaml

@@ -127,6 +127,10 @@ placement_policy:
   # Region to placement policy mapping json file.
   # Path to container policy mapping. The same as '--container-policy' flag for authmate
   region_mapping: /path/to/container/policy.json
+  locations:
+    REP-1: "REP 1"
+    REP-3: "REP 3"
+    complex: "REP 1 IN X CBF 1 SELECT 1 FROM * AS X"
 
 # CORS
 # value of Access-Control-Max-Age header if this value is not set in a rule. Has an int type.

docs/configuration.md

@@ -257,10 +257,11 @@ placement_policy:
   region_mapping: /path/to/mapping/rules.json
 ```
 
-| Parameter        | Type     | SIGHUP reload | Default value | Description                                                                                                                                                                                                       |
-|------------------|----------|---------------|---------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `default`        | `string` | yes           | `REP 3`       | Default policy of placing containers in NeoFS. If a user sends a request `CreateBucket` and doesn't define policy for placing of a container in NeoFS, the S3 Gateway will put the container with default policy. |
-| `region_mapping` | `string` | yes           |               | Path to file that maps aws `LocationContraint` values to NeoFS placement policy. The similar to `--container-policy` flag in `neofs-s3-authmate` util, see in [docs](./authmate.md#containers-policy)             |
+| Parameter        | Type     | SIGHUP reload | Default value | Description                                                                                                                                                                                                                                  |
+|------------------|----------|---------------|---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `default`        | `string` | yes           | `REP 3`       | Default policy of placing containers in NeoFS. If a user sends a request `CreateBucket` and doesn't define policy for placing of a container in NeoFS, the S3 Gateway will put the container with default policy.                            |
+| `region_mapping` | `string` | yes           |               | Path to file that maps aws `LocationConstraint` values to NeoFS placement policy. The similar to `--container-policy` flag in `neofs-s3-authmate` util, see in [docs](./authmate.md#containers-policy)                                       |
+| `locations`      | `map`    | yes           |               | Hashtable that maps aws LocationConstraint values to NeoFS placement policy. It's similar to the region_mapping config option, but allows for in-place configuration and extends/overrides values from region_mapping file if it's provided. |
 
 File for `region_mapping` must contain something like this:
 
@@ -272,6 +273,15 @@ File for `region_mapping` must contain something like this:
 }
 ```
 
+The option `placement_policy.regions` may look like:
+```yaml
+placement_policy:
+  locations:
+    REP-1: "REP 1"
+    REP-3: "REP 3"
+    complex: "REP 1 IN X CBF 1 SELECT 1 FROM * AS X"
+```
+
 **Note:** on SIGHUP reload policies will be updated only if both parameters are valid. 
 So if you change `default` to some valid value and set invalid path in `region_mapping` the `default` value won't be changed.