From 0935df7696a51e0bbcabab3dc6b1b804b9513583 Mon Sep 17 00:00:00 2001 From: Alan Brault Date: Wed, 23 Jan 2013 16:22:23 -0500 Subject: [PATCH 1/4] Add the ability for file conveyor to properly handle secure urls, signed urls and expiring urls from CloudFront/S3 properly. --- fileconveyor/transporters/transporter_cf.py | 15 +++++++-- fileconveyor/transporters/transporter_s3.py | 37 +++++++++++++++++++-- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/fileconveyor/transporters/transporter_cf.py b/fileconveyor/transporters/transporter_cf.py index 8e1f592..f5d8299 100644 --- a/fileconveyor/transporters/transporter_cf.py +++ b/fileconveyor/transporters/transporter_cf.py @@ -9,13 +9,24 @@ class TransporterCF(TransporterS3): name = 'CF' - valid_settings = ImmutableSet(["access_key_id", "secret_access_key", "bucket_name", "distro_domain_name", "bucket_prefix"]) + valid_settings = ImmutableSet(["access_key_id", "secret_access_key", "bucket_name", "distro_domain_name", "bucket_prefix", + "use_signed_urls", "use_secure_urls", "set_url_expiration", "secure_distro_domain_name"]) required_settings = ImmutableSet(["access_key_id", "secret_access_key", "bucket_name", "distro_domain_name"]) def __init__(self, settings, callback, error_callback, parent_logger=None): - return TransporterS3.__init__(self, settings, callback, error_callback, parent_logger) + # Fill out defaults if necessary. + configured_settings = Set(self.settings.keys()) + + if self.settings["use_secure_urls"].lower() == "true": + if not "secure_distro_domain_name" in configured_settings: + if self.settings["distro_domain_name"].partition('.')[2] == "cloudfront.net": + self.settings["secure_distro_domain_name"] = self.settings["distro_domain_name"] + else: + self.logger.error("You must set 'secure_distro_domain_name' when you have 'use_secure_urls' set but are not using a cloudfront.net host for 'distro_domain_name'") + raise InvalidSettingError + return TransporterS3.__init__(self, settings, callback, error_callback, parent_logger) def alter_url(self, url): return url.replace( diff --git a/fileconveyor/transporters/transporter_s3.py b/fileconveyor/transporters/transporter_s3.py index 71cd5fc..364d19c 100644 --- a/fileconveyor/transporters/transporter_s3.py +++ b/fileconveyor/transporters/transporter_s3.py @@ -9,7 +9,8 @@ class TransporterS3(Transporter): name = 'S3' - valid_settings = ImmutableSet(["access_key_id", "secret_access_key", "bucket_name", "bucket_prefix"]) + valid_settings = ImmutableSet(["access_key_id", "secret_access_key", "bucket_name", "bucket_prefix", + "use_signed_urls", "use_secure_urls", "set_url_expiration"]) required_settings = ImmutableSet(["access_key_id", "secret_access_key", "bucket_name"]) headers = { 'Expires': 'Tue, 20 Jan 2037 03:00:00 GMT', # UNIX timestamps will stop working somewhere in 2038. @@ -26,6 +27,34 @@ def __init__(self, settings, callback, error_callback, parent_logger=None): if not "bucket_prefix" in configured_settings: self.settings["bucket_prefix"] = "" + # Continue default behavior of S3BotoStorage + # if the user has not defined these in the + # configuration file. + if not "use_signed_urls" in configured_settings: + self.settings["use_signed_urls"] = "true" + + if not "use_secure_urls" in configured_settings: + self.settings["use_secure_urls"] = "true" + + if not "set_url_expiration" in configured_settings: + self.settings["set_url_expiration"] = 3600 + + # set_url_expiration must be an integer, if it is not, set to default. + if not isinstance(self.settings["set_url_expiration"], int): + self.settings["set_url_expiration"] = 3600 + + if self.settings["use_signed_urls"].lower() == "true": + USE_QUERYSTRING_AUTH=True + else: + USE_QUERYSTRING_AUTH=False + + if self.settings["use_secure_urls"].lower() == "true": + USE_SECURE_URLS=True + USE_URL_PROTOCOL="https" + else: + USE_SECURE_URLS=False + USE_URL_PROTOCOL="http" + # Map the settings to the format expected by S3Storage. try: self.storage = S3BotoStorage( @@ -34,7 +63,11 @@ def __init__(self, settings, callback, error_callback, parent_logger=None): self.settings["secret_access_key"].encode('utf-8'), "public-read", "public-read", - self.__class__.headers + self.__class__.headers, + querystring_auth=USE_QUERYSTRING_AUTH, + querystring_expire=self.settings["set_url_expiration"], + secure_urls=USE_SECURE_URLS, + url_protocol=USE_URL_PROTOCOL ) except Exception, e: raise ConnectionError(e) From ffdec3609566eb441c1d20856cefea4e92e0ea62 Mon Sep 17 00:00:00 2001 From: Alan Brault Date: Thu, 24 Jan 2013 11:17:24 -0500 Subject: [PATCH 2/4] Forgot to tell alter_url about the new use_secure_urls logic --- fileconveyor/transporters/transporter_cf.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/fileconveyor/transporters/transporter_cf.py b/fileconveyor/transporters/transporter_cf.py index f5d8299..6dc8b8f 100644 --- a/fileconveyor/transporters/transporter_cf.py +++ b/fileconveyor/transporters/transporter_cf.py @@ -29,10 +29,16 @@ def __init__(self, settings, callback, error_callback, parent_logger=None): return TransporterS3.__init__(self, settings, callback, error_callback, parent_logger) def alter_url(self, url): - return url.replace( - self.settings["bucket_prefix"] + self.settings["bucket_name"] + ".s3.amazonaws.com", - self.settings["distro_domain_name"] - ) + if self.settings["use_secure_urls"].lower() == "true": + return url.replace( + self.settings["bucket_prefix"] + self.settings["bucket_name"] + ".s3.amazonaws.com", + self.settings["secure_distro_domain_name"] + ) + else: + return url.replace( + self.settings["bucket_prefix"] + self.settings["bucket_name"] + ".s3.amazonaws.com", + self.settings["distro_domain_name"] + ) def create_distribution(access_key_id, secret_access_key, origin, comment="", cnames=None): From 6f1d1a247f02b1a105042f13eba6aef4cd737752 Mon Sep 17 00:00:00 2001 From: yusufhm Date: Wed, 30 Jan 2013 22:44:38 +0400 Subject: [PATCH 3/4] Update fileconveyor/transporters/transporter_cf.py I had to call TransporterS3's __init__(), otherwise I got an error of settings not existing.. --- fileconveyor/transporters/transporter_cf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fileconveyor/transporters/transporter_cf.py b/fileconveyor/transporters/transporter_cf.py index 6dc8b8f..34d6957 100644 --- a/fileconveyor/transporters/transporter_cf.py +++ b/fileconveyor/transporters/transporter_cf.py @@ -16,6 +16,7 @@ class TransporterCF(TransporterS3): def __init__(self, settings, callback, error_callback, parent_logger=None): # Fill out defaults if necessary. + TransporterS3.__init__(self, settings, callback, error_callback, parent_logger) configured_settings = Set(self.settings.keys()) if self.settings["use_secure_urls"].lower() == "true": From 32f737e009aba6dc1b0c6f2fe65810ecf356f103 Mon Sep 17 00:00:00 2001 From: yusufhm Date: Thu, 31 Jan 2013 00:29:40 +0400 Subject: [PATCH 4/4] Update fileconveyor/transporters/transporter_s3.py I had to comment out the url_protocol argument due to which the call to S3BotoStorage failed, saying it did not recognize that argument. --- fileconveyor/transporters/transporter_s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fileconveyor/transporters/transporter_s3.py b/fileconveyor/transporters/transporter_s3.py index 364d19c..ad865d2 100644 --- a/fileconveyor/transporters/transporter_s3.py +++ b/fileconveyor/transporters/transporter_s3.py @@ -67,7 +67,7 @@ def __init__(self, settings, callback, error_callback, parent_logger=None): querystring_auth=USE_QUERYSTRING_AUTH, querystring_expire=self.settings["set_url_expiration"], secure_urls=USE_SECURE_URLS, - url_protocol=USE_URL_PROTOCOL + #url_protocol=USE_URL_PROTOCOL ) except Exception, e: raise ConnectionError(e)