From decbbf3a47b60a13f55f9d756ee1b0169a323c11 Mon Sep 17 00:00:00 2001 From: Callum Dickinson Date: Thu, 11 Jul 2024 12:17:32 +1200 Subject: [PATCH] Swift: Allow configuring container storage policies This adds the `swift_storage_policy` configuration option for setting the storage policy Gnocchi will use when creating new Swift containers. When unset the storage policy will not be configured when creating containers, in which case Swift will use the default storage policy. --- gnocchi/incoming/swift.py | 11 +++++++++-- gnocchi/storage/swift.py | 10 ++++++++++ gnocchi/tests/base.py | 15 +++++++++++---- .../swift-storage-policy-fa2d73d8714e3a03.yaml | 8 ++++++++ 4 files changed, 38 insertions(+), 6 deletions(-) create mode 100644 releasenotes/notes/swift-storage-policy-fa2d73d8714e3a03.yaml diff --git a/gnocchi/incoming/swift.py b/gnocchi/incoming/swift.py index bc1429282..9cb728b05 100644 --- a/gnocchi/incoming/swift.py +++ b/gnocchi/incoming/swift.py @@ -36,6 +36,11 @@ class SwiftStorage(incoming.IncomingDriver): def __init__(self, conf, greedy=True): super(SwiftStorage, self).__init__(conf) self.swift = swift.get_connection(conf) + self._put_container_headers = {} + if conf.swift_storage_policy: + self._put_container_headers["X-Storage-Policy"] = ( + conf.swift_storage_policy + ) def __str__(self): return self.__class__.__name__ @@ -45,11 +50,13 @@ def _get_storage_sacks(self): return json.loads(data)[self.CFG_SACKS] def set_storage_settings(self, num_sacks): - self.swift.put_container(self.CFG_PREFIX) + self.swift.put_container(self.CFG_PREFIX, + headers=self._put_container_headers) self.swift.put_object(self.CFG_PREFIX, self.CFG_PREFIX, json.dumps({self.CFG_SACKS: num_sacks})) for sack in self.iter_sacks(): - self.swift.put_container(str(sack)) + self.swift.put_container(str(sack), + headers=self._put_container_headers) def remove_sacks(self): for sack in self.iter_sacks(): diff --git a/gnocchi/storage/swift.py b/gnocchi/storage/swift.py index 4ade3855a..805823a24 100644 --- a/gnocchi/storage/swift.py +++ b/gnocchi/storage/swift.py @@ -67,6 +67,10 @@ cfg.StrOpt('swift_container_prefix', default='gnocchi', help='Prefix to namespace metric containers.'), + cfg.StrOpt('swift_storage_policy', + default=None, + help='Storage policy to use when creating containers. ' + 'When unset, the default Swift storage policy is used.'), cfg.StrOpt('swift_endpoint_type', default='publicURL', help='Endpoint type to connect to Swift',), @@ -93,6 +97,11 @@ def __init__(self, conf): super(SwiftStorage, self).__init__(conf) self.swift = swift.get_connection(conf) self._container_prefix = conf.swift_container_prefix + self._put_container_headers = {} + if conf.swift_storage_policy: + self._put_container_headers["X-Storage-Policy"] = ( + conf.swift_storage_policy + ) def __str__(self): return "%s: %s" % (self.__class__.__name__, self._container_prefix) @@ -112,6 +121,7 @@ def _create_metric(self, metric): # TODO(jd) A container per user in their account? resp = {} self.swift.put_container(self._container_name(metric), + headers=self._put_container_headers, response_dict=resp) # put_container() should return 201 Created; if it returns 204, that # means the metric was already created! diff --git a/gnocchi/tests/base.py b/gnocchi/tests/base.py index c059dd4a6..c25d8d3e8 100644 --- a/gnocchi/tests/base.py +++ b/gnocchi/tests/base.py @@ -65,26 +65,32 @@ def skip_if_not_implemented(*args, **kwargs): class FakeSwiftClient(object): def __init__(self, *args, **kwargs): self.kvs = {} + self.headers = {} - def put_container(self, container, response_dict=None): + def put_container(self, container, headers=None, response_dict=None): if response_dict is not None: if container in self.kvs: response_dict['status'] = 204 else: response_dict['status'] = 201 self.kvs[container] = {} + self.headers[container] = ( + {header.lower(): value for header, value in headers.items()} + if headers + else {} + ) def get_container(self, container, delimiter=None, path=None, full_listing=False, limit=None): try: - container = self.kvs[container] + container_objs = self.kvs[container] except KeyError: raise swexc.ClientException("No such container", http_status=404) files = [] directories = set() - for k, v in container.copy().items(): + for k, v in container_objs.items(): if path and not k.startswith(path): continue @@ -109,7 +115,8 @@ def get_container(self, container, delimiter=None, # otherwise. end = 1 - return ({'x-container-object-count': len(container.keys())}, + return ({**self.headers[container], + 'x-container-object-count': len(container_objs.keys())}, (files + list(directories))[:end]) def put_object(self, container, key, obj): diff --git a/releasenotes/notes/swift-storage-policy-fa2d73d8714e3a03.yaml b/releasenotes/notes/swift-storage-policy-fa2d73d8714e3a03.yaml new file mode 100644 index 000000000..57d0bc06c --- /dev/null +++ b/releasenotes/notes/swift-storage-policy-fa2d73d8714e3a03.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + The Swift storage driver can now be configured to create containers + with a custom storage policy using the ``swift_storage_policy`` + option. This allows for overriding the storage policy when using + the default policy is not desirable, e.g. to limit object replication + to a single region when multi-region replication is the default.