From 07ab62a1a3b17b3c61ef3d81bde1419b1d1e7956 Mon Sep 17 00:00:00 2001 From: jackyalbo Date: Mon, 18 Nov 2024 16:30:59 +0200 Subject: [PATCH] CORS Signed-off-by: jackyalbo --- config.js | 16 ++--- src/api/bucket_api.js | 64 ++++++++++++++++++- src/api/common_api.js | 44 +++++++++++++ src/endpoint/s3/ops/s3_delete_bucket_cors.js | 5 +- src/endpoint/s3/ops/s3_get_bucket_cors.js | 19 ++++-- src/endpoint/s3/ops/s3_put_bucket.js | 13 ++++ src/endpoint/s3/ops/s3_put_bucket_cors.js | 19 ++++-- src/endpoint/s3/s3_errors.js | 5 ++ src/endpoint/s3/s3_rest.js | 20 +++--- src/sdk/bucketspace_fs.js | 41 ++++++++++++ src/sdk/bucketspace_nb.js | 27 +++++++- src/sdk/nb.d.ts | 4 ++ src/sdk/object_sdk.js | 48 +++++++++++--- src/server/common_services/auth_server.js | 4 +- src/server/system_services/bucket_server.js | 52 +++++++++++++-- .../system_services/schemas/bucket_schema.js | 6 +- .../schemas/nsfs_bucket_schema.js | 5 +- .../run_ceph_nsfs_test_on_test_container.sh | 1 + .../run_ceph_test_on_test_container.sh | 3 + .../nsfs_s3_tests_black_list.txt | 6 -- .../s3-tests-lists/s3_tests_black_list.txt | 9 ++- .../s3-tests-lists/s3_tests_pending_list.txt | 4 -- src/util/http_utils.js | 54 ++++++++++++---- 23 files changed, 393 insertions(+), 76 deletions(-) diff --git a/config.js b/config.js index e6540e8a43..855de9bc58 100644 --- a/config.js +++ b/config.js @@ -64,7 +64,7 @@ config.BUFFERS_MEM_LIMIT_MIN = 32 * 1024 * 1024; // just some workable minimum s config.BUFFERS_MEM_LIMIT_MAX = 4 * 1024 * 1024 * 1024; config.BUFFERS_MEM_LIMIT = Math.min( config.BUFFERS_MEM_LIMIT_MAX, - Math.max(Math.floor(config.CONTAINER_MEM_LIMIT / 4), config.BUFFERS_MEM_LIMIT_MIN, ) + Math.max(Math.floor(config.CONTAINER_MEM_LIMIT / 4), config.BUFFERS_MEM_LIMIT_MIN) ); //////////////////////// @@ -152,9 +152,8 @@ config.ENDPOINT_HTTP_SERVER_REQUEST_TIMEOUT = 300 * 1000; config.ENDPOINT_HTTP_SERVER_KEEPALIVE_TIMEOUT = 5 * 1000; config.ENDPOINT_HTTP_MAX_REQUESTS_PER_SOCKET = 0; // 0 = no limit -// For now we enable fixed CORS for all buckets -// but this should become a setting per bucket which is configurable -// with the s3 put-bucket-cors api. +// For now we enable fixed CORS only for sts +// for S3 per bucket is configurabl with the s3 put-bucket-cors api. // note that browsers do not really allow origin=* with credentials, // but we just allow both from our side for simplicity. config.S3_CORS_ENABLED = true; @@ -502,6 +501,7 @@ config.LOG_COLOR_ENABLED = process.env.NOOBAA_LOG_COLOR ? process.env.NOOBAA_LOG // TEST Mode config.test_mode = false; +config.allow_anonymous_access_in_test = false; // used for emulating ACL='public-read' for ceph-s3 tests // On Premise NVA params config.on_premise = { @@ -753,11 +753,11 @@ config.NSFS_BUF_POOL_MEM_LIMIT_XS = Math.min(Math.floor(config.NSFS_MAX_MEM_SIZE config.NSFS_BUF_POOL_MEM_LIMIT_S = Math.min(Math.floor(config.NSFS_MAX_MEM_SIZE_S / config.NSFS_BUF_SIZE_S), config.NSFS_WANTED_BUFFERS_NUMBER) * config.NSFS_BUF_SIZE_S; // Semaphore size will give 90% of remainning memory to large buffer size, 10% to medium -config.NSFS_BUF_POOL_MEM_LIMIT_M = range_utils.align_down((config.BUFFERS_MEM_LIMIT - - config.NSFS_BUF_POOL_MEM_LIMIT_S - config.NSFS_BUF_POOL_MEM_LIMIT_XS) * 0.1, +config.NSFS_BUF_POOL_MEM_LIMIT_M = range_utils.align_down( + (config.BUFFERS_MEM_LIMIT - config.NSFS_BUF_POOL_MEM_LIMIT_S - config.NSFS_BUF_POOL_MEM_LIMIT_XS) * 0.1, config.NSFS_BUF_SIZE_M); -config.NSFS_BUF_POOL_MEM_LIMIT_L = range_utils.align_down((config.BUFFERS_MEM_LIMIT - - config.NSFS_BUF_POOL_MEM_LIMIT_S - config.NSFS_BUF_POOL_MEM_LIMIT_XS) * 0.9, +config.NSFS_BUF_POOL_MEM_LIMIT_L = range_utils.align_down( + (config.BUFFERS_MEM_LIMIT - config.NSFS_BUF_POOL_MEM_LIMIT_S - config.NSFS_BUF_POOL_MEM_LIMIT_XS) * 0.9, config.NSFS_BUF_SIZE_L); config.NSFS_BUF_WARMUP_SPARSE_FILE_READS = true; diff --git a/src/api/bucket_api.js b/src/api/bucket_api.js index 8e53fb6bb6..e06b09b46a 100644 --- a/src/api/bucket_api.js +++ b/src/api/bucket_api.js @@ -343,7 +343,7 @@ module.exports = { notifications: { type: 'array', items: { - $ref: 'common_api#/definitions/bucket_notification' + $ref: 'common_api#/definitions/bucket_notification' } } } @@ -890,6 +890,65 @@ module.exports = { system: ['admin', 'user'] } }, + + get_bucket_cors: { + method: 'GET', + params: { + type: 'object', + required: ['name'], + properties: { + name: { + $ref: 'common_api#/definitions/bucket_name' + }, + }, + }, + reply: { + type: 'object', + properties: { + cors: { + $ref: 'common_api#/definitions/bucket_cors_configuration' + } + } + }, + auth: { + system: ['admin', 'user'] + } + }, + + put_bucket_cors: { + method: 'PUT', + params: { + type: 'object', + required: ['name', 'cors_rules'], + properties: { + name: { + $ref: 'common_api#/definitions/bucket_name' + }, + cors_rules: { + $ref: 'common_api#/definitions/bucket_cors_configuration' + }, + }, + }, + auth: { + system: ['admin', 'user'] + } + }, + + delete_bucket_cors: { + method: 'DELETE', + params: { + type: 'object', + required: ['name'], + properties: { + name: { + $ref: 'common_api#/definitions/bucket_name' + }, + }, + }, + auth: { + system: ['admin', 'user'] + } + }, }, definitions: { @@ -1200,6 +1259,9 @@ module.exports = { items: { $ref: 'common_api#/definitions/bucket_notification' } + }, + cors_configuration_rules: { + $ref: 'common_api#/definitions/bucket_cors_configuration', } } }, diff --git a/src/api/common_api.js b/src/api/common_api.js index 454f14811a..671f520cca 100644 --- a/src/api/common_api.js +++ b/src/api/common_api.js @@ -271,6 +271,50 @@ module.exports = { } }, + bucket_cors_configuration: { + type: 'array', + items: { + $ref: '#/definitions/bucket_cors_rule' + } + }, + + bucket_cors_rule: { + type: 'object', + required: ['allowed_methods', 'allowed_origins'], + properties: { + id: { + type: 'string' + }, + allowed_methods: { + type: 'array', + items: { + type: 'string' + } + }, + allowed_origins: { + type: 'array', + items: { + type: 'string' + } + }, + allowed_headers: { + type: 'array', + items: { + type: 'string' + } + }, + expose_headers: { + type: 'array', + items: { + type: 'string' + } + }, + max_age_seconds: { + type: 'integer' + }, + } + }, + bucket_policy_principal: { anyOf: [{ wrapper: SensitiveString diff --git a/src/endpoint/s3/ops/s3_delete_bucket_cors.js b/src/endpoint/s3/ops/s3_delete_bucket_cors.js index 042b4d4fbb..42e3265a1b 100644 --- a/src/endpoint/s3/ops/s3_delete_bucket_cors.js +++ b/src/endpoint/s3/ops/s3_delete_bucket_cors.js @@ -5,8 +5,9 @@ * http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketDELETEcors.html */ async function delete_bucket_cors(req) { - await req.object_sdk.read_bucket({ name: req.params.bucket }); - // TODO S3 delete_bucket_cors not implemented + await req.object_sdk.delete_bucket_cors({ + name: req.params.bucket, + }); } module.exports = { diff --git a/src/endpoint/s3/ops/s3_get_bucket_cors.js b/src/endpoint/s3/ops/s3_get_bucket_cors.js index 74d5ed422b..e58ac66099 100644 --- a/src/endpoint/s3/ops/s3_get_bucket_cors.js +++ b/src/endpoint/s3/ops/s3_get_bucket_cors.js @@ -1,14 +1,25 @@ /* Copyright (C) 2016 NooBaa */ 'use strict'; +const S3Error = require('../s3_errors').S3Error; + /** * http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETcors.html */ async function get_bucket_cors(req) { - await req.object_sdk.read_bucket({ name: req.params.bucket }); - return { - CORSConfiguration: '' - }; + const reply = await req.object_sdk.get_bucket_cors({ name: req.params.bucket }); + if (!reply.cors.length) throw new S3Error(S3Error.NoSuchCORSConfiguration); + const cors_rules = reply.cors.map(rule => { + const new_rule = []; + new_rule.push(...rule.allowed_methods.map(m => ({ AllowedMethod: m }))); + new_rule.push(...rule.allowed_origins.map(o => ({ AllowedOrigin: o }))); + if (rule.allowed_headers) new_rule.push(...rule.allowed_headers.map(h => ({ AllowedHeader: h }))); + if (rule.expose_headers) new_rule.push(...rule.expose_headers.map(e => ({ ExposeHeader: e }))); + if (rule.id) new_rule.push({ ID: rule.id }); + if (rule.max_age_seconds) new_rule.push({ MaxAgeSeconds: rule.max_age_seconds }); + return { CORSRule: new_rule }; + }); + return { CORSConfiguration: cors_rules.length ? cors_rules : '' }; } module.exports = { diff --git a/src/endpoint/s3/ops/s3_put_bucket.js b/src/endpoint/s3/ops/s3_put_bucket.js index fd588dafdd..c930c1baaf 100644 --- a/src/endpoint/s3/ops/s3_put_bucket.js +++ b/src/endpoint/s3/ops/s3_put_bucket.js @@ -2,6 +2,7 @@ 'use strict'; const config = require('../../../../config'); + /** * http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUT.html */ @@ -9,6 +10,18 @@ async function put_bucket(req, res) { const lock_enabled = config.WORM_ENABLED ? req.headers['x-amz-bucket-object-lock-enabled'] && req.headers['x-amz-bucket-object-lock-enabled'].toUpperCase() === 'TRUE' : undefined; await req.object_sdk.create_bucket({ name: req.params.bucket, lock_enabled: lock_enabled }); + if (config.allow_anonymous_access_in_test && req.headers['x-amz-acl'] === 'public-read') { // For now we will enable only for tests + const policy = { + Version: '2012-10-17', + Statement: [{ + Effect: 'Allow', + Principal: { AWS: ["*"] }, + Action: ['s3:GetObject', 's3:ListBucket'], + Resource: ['arn:aws:s3:::*'] + }] + }; + await req.object_sdk.put_bucket_policy({ name: req.params.bucket, policy }); + } res.setHeader('Location', '/' + req.params.bucket); } diff --git a/src/endpoint/s3/ops/s3_put_bucket_cors.js b/src/endpoint/s3/ops/s3_put_bucket_cors.js index 30efa299d7..ec83c8a1bf 100644 --- a/src/endpoint/s3/ops/s3_put_bucket_cors.js +++ b/src/endpoint/s3/ops/s3_put_bucket_cors.js @@ -1,15 +1,26 @@ /* Copyright (C) 2016 NooBaa */ 'use strict'; -const S3Error = require('../s3_errors').S3Error; +const _ = require('lodash'); /** * http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTcors.html */ async function put_bucket_cors(req) { - await req.object_sdk.read_bucket({ name: req.params.bucket }); - // TODO S3 put_bucket_cors not implemented - throw new S3Error(S3Error.NotImplemented); + const cors_rules = req.body.CORSConfiguration.CORSRule.map(rule => + _.omitBy({ + allowed_headers: rule.AllowedHeader, + allowed_methods: rule.AllowedMethod, + allowed_origins: rule.AllowedOrigin, + expose_headers: rule.ExposeHeader, + id: rule.ID, + max_age_seconds: rule.MaxAgeSeconds, + }, _.isUndefined) + ); + await req.object_sdk.put_bucket_cors({ + name: req.params.bucket, + cors_rules + }); } module.exports = { diff --git a/src/endpoint/s3/s3_errors.js b/src/endpoint/s3/s3_errors.js index 2800a8d538..13965c1398 100644 --- a/src/endpoint/s3/s3_errors.js +++ b/src/endpoint/s3/s3_errors.js @@ -339,6 +339,11 @@ S3Error.NoSuchLifecycleConfiguration = Object.freeze({ message: 'The lifecycle configuration does not exist.', http_code: 404, }); +S3Error.NoSuchCORSConfiguration = Object.freeze({ + code: 'NoSuchCORSConfiguration', + message: 'The specified bucket does not have a CORS configuration.', + http_code: 404, +}); S3Error.NoSuchUpload = Object.freeze({ code: 'NoSuchUpload', message: 'The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.', diff --git a/src/endpoint/s3/s3_rest.js b/src/endpoint/s3/s3_rest.js index 9e753cb984..48ce6a00b4 100755 --- a/src/endpoint/s3/s3_rest.js +++ b/src/endpoint/s3/s3_rest.js @@ -83,14 +83,6 @@ async function handle_request(req, res) { http_utils.validate_server_ip_whitelist(req); http_utils.set_amz_headers(req, res); - http_utils.set_cors_headers_s3(req, res); - - if (req.method === 'OPTIONS') { - dbg.log1('OPTIONS!'); - res.statusCode = 200; - res.end(); - return; - } const headers_options = { ErrorClass: S3Error, @@ -115,6 +107,18 @@ async function handle_request(req, res) { } const op_name = parse_op_name(req); + const cors = req.params.bucket && await req.object_sdk.read_bucket_sdk_cors_info(req.params.bucket); + + http_utils.set_cors_headers_s3(req, res, cors); + + if (req.method === 'OPTIONS') { + dbg.log1('OPTIONS!'); + const error_code = req.headers.origin && req.headers['access-control-request-method'] ? 403 : 400; + const res_headers = res.getHeaders(); // We will check if we found a matching rule - if no we will return error_code + res.statusCode = res_headers['access-control-allow-origin'] && res_headers['access-control-allow-methods'] ? 200 : error_code; + res.end(); + return; + } const op = s3_ops[op_name]; if (!op || !op.handler) { dbg.error('S3 NotImplemented', op_name, req.method, req.originalUrl); diff --git a/src/sdk/bucketspace_fs.js b/src/sdk/bucketspace_fs.js index e3d3788b2b..6f07c5ed7c 100644 --- a/src/sdk/bucketspace_fs.js +++ b/src/sdk/bucketspace_fs.js @@ -697,6 +697,47 @@ class BucketSpaceFS extends BucketSpaceSimpleFS { } } + //////////////////// + // BUCKET CORS // + //////////////////// + + async put_bucket_cors(params) { + try { + const { name, cors_rules } = params; + dbg.log0('BucketSpaceFS.put_bucket_cors: Bucket name', name, ", cors configuration ", cors_rules); + const bucket = await this.config_fs.get_bucket_by_name(name); + bucket.cors_configuration_rules = cors_rules; + await this.config_fs.update_bucket_config_file(bucket); + } catch (error) { + throw translate_error_codes(error, entity_enum.BUCKET); + } + } + + async delete_bucket_cors(params) { + try { + const { name } = params; + dbg.log0('BucketSpaceFS.delete_bucket_cors: Bucket name', name); + const bucket = await this.config_fs.get_bucket_by_name(name); + delete bucket.cors_configuration_rules; + await this.config_fs.update_bucket_config_file(bucket); + } catch (err) { + throw translate_error_codes(err, entity_enum.BUCKET); + } + } + + async get_bucket_cors(params) { + try { + const { name } = params; + dbg.log0('BucketSpaceFS.get_bucket_cors: Bucket name', name); + const bucket = await this.config_fs.get_bucket_by_name(name); + return { + cors: bucket.cors_configuration_rules || [], + }; + } catch (error) { + throw translate_error_codes(error, entity_enum.BUCKET); + } + } + ///////////////////////// // DEFAULT OBJECT LOCK // ///////////////////////// diff --git a/src/sdk/bucketspace_nb.js b/src/sdk/bucketspace_nb.js index d3b2b3a930..88b15767da 100644 --- a/src/sdk/bucketspace_nb.js +++ b/src/sdk/bucketspace_nb.js @@ -33,7 +33,7 @@ class BucketSpaceNB { //////////// async list_buckets(params, object_sdk) { - const { buckets, continuation_token} = (await this.rpc_client.bucket.list_buckets(params)); + const { buckets, continuation_token } = (await this.rpc_client.bucket.list_buckets(params)); const has_access_buckets = (await P.all(_.map( buckets, @@ -44,7 +44,7 @@ class BucketSpaceNB { object_sdk.has_non_nsfs_bucket_access(object_sdk.requesting_account, ns); return has_access_to_bucket && bucket; }))).filter(bucket => bucket); - return { buckets: has_access_buckets, continuation_token}; + return { buckets: has_access_buckets, continuation_token }; } async read_bucket(params) { @@ -247,6 +247,29 @@ class BucketSpaceNB { }); } + //////////////////// + // BUCKET CORS // + //////////////////// + + async put_bucket_cors(params) { + return this.rpc_client.bucket.put_bucket_cors({ + name: params.name, + cors_rules: params.cors_rules + }); + } + + async delete_bucket_cors(params) { + return this.rpc_client.bucket.delete_bucket_cors({ + name: params.name + }); + } + + async get_bucket_cors(params) { + return this.rpc_client.bucket.get_bucket_cors({ + name: params.name + }); + } + ///////////////////////// // DEFAULT OBJECT LOCK // ///////////////////////// diff --git a/src/sdk/nb.d.ts b/src/sdk/nb.d.ts index a98071294a..29abd76fb4 100644 --- a/src/sdk/nb.d.ts +++ b/src/sdk/nb.d.ts @@ -860,6 +860,10 @@ interface BucketSpace { put_bucket_notification(params: object): Promise; get_bucket_notification(params: object): Promise; + put_bucket_cors(params: object): Promise; + delete_bucket_cors(params: object): Promise; + get_bucket_cors(params: object): Promise; + get_object_lock_configuration(params: object, object_sdk: ObjectSDK): Promise; put_object_lock_configuration(params: object, object_sdk: ObjectSDK): Promise; diff --git a/src/sdk/object_sdk.js b/src/sdk/object_sdk.js index a722655817..864e9a1ae6 100644 --- a/src/sdk/object_sdk.js +++ b/src/sdk/object_sdk.js @@ -67,12 +67,12 @@ const dn_cache = new LRUCache({ /** * Set type for the generic template * @param {{ - * distinguished_name: string; - * }} params - */ + * distinguished_name: string; + * }} params + */ make_key: ({ distinguished_name }) => distinguished_name, load: async ({ distinguished_name }) => native_fs_utils.get_user_by_distinguished_name({ distinguished_name }), - }); +}); const MULTIPART_NAMESPACES = [ 'NET_STORAGE' @@ -210,6 +210,16 @@ class ObjectSDK { return bucket_namespace_cache.get_with_cache({ sdk: this, name }); } + async read_bucket_sdk_cors_info(name) { + try { + const { bucket } = await bucket_namespace_cache.get_with_cache({ sdk: this, name }); + return bucket.cors_configuration_rules; + } catch (error) { + if (error.rpc_code === 'NO_SUCH_BUCKET') return undefined; + throw error; + } + } + async load_requesting_account(req) { try { const token = this.get_auth_token(); @@ -342,8 +352,7 @@ class ObjectSDK { return { ns: this._setup_single_namespace( bucket.namespace.read_resources[0], - bucket._id, - { + bucket._id, { versioning: bucket.bucket_info && bucket.bucket_info.versioning, force_md5_etag: bucket.force_md5_etag }, @@ -400,8 +409,8 @@ class ObjectSDK { write_resource: wr, read_resources: _.map(rr, it => ( it.resource.endpoint_type === 'MULTIPART' ? - it.ns : - this._setup_single_namespace(it) + it.ns : + this._setup_single_namespace(it) )) }, active_triggers: bucket.active_triggers @@ -1090,6 +1099,25 @@ class ObjectSDK { return bucket.notifications; } + //////////////////// + // BUCKET CORS // + //////////////////// + + async put_bucket_cors(params) { + const bs = this._get_bucketspace(); + return bs.put_bucket_cors(params); + } + + async delete_bucket_cors(params) { + const bs = this._get_bucketspace(); + return bs.delete_bucket_cors(params); + } + + async get_bucket_cors(params) { + const bs = this._get_bucketspace(); + return bs.get_bucket_cors(params); + } + //////////////////// // OBJECT LOCK // //////////////////// @@ -1145,13 +1173,13 @@ class ObjectSDK { const ns = await this._get_bucket_namespace(params.bucket); if (ns.get_object_attributes) { return ns.get_object_attributes(params, this); - } else { + } else { // fallback to calling get_object_md without attributes params dbg.warn('namespace does not implement get_object_attributes action, fallback to read_object_md'); const md_params = { ...params }; delete md_params.attributes; // not part of the schema of read_object_md return ns.read_object_md(md_params, this); - } + } } } diff --git a/src/server/common_services/auth_server.js b/src/server/common_services/auth_server.js index 76cfa58012..93160a1238 100644 --- a/src/server/common_services/auth_server.js +++ b/src/server/common_services/auth_server.js @@ -668,7 +668,7 @@ function _get_auth_info(account, system, authorized_by, role, extra) { * @param {Record} account requesting account * @param {string} action s3 bucket action (lowercased only) * @param {string} bucket_path s3 bucket path (must start from "/") - * @returns {boolean} true if the account has permission to perform the action on the bucket + * @returns {Promise} true if the account has permission to perform the action on the bucket */ async function has_bucket_action_permission(bucket, account, action, bucket_path = "") { dbg.log1('has_bucket_action_permission:', bucket.name, account.email, bucket.owner_account.email); @@ -703,7 +703,7 @@ async function has_bucket_action_permission(bucket, account, action, bucket_path * @param {Record} bucket bucket * @param {string} action s3 action (lowercased) * @param {string} bucket_path bucket path - * @returns {boolean} true if the bucket is configured to serve anonymous requests + * @returns {Promise} true if the bucket is configured to serve anonymous requests */ async function has_bucket_anonymous_permission(bucket, action, bucket_path = "") { const bucket_policy = bucket.s3_policy; diff --git a/src/server/system_services/bucket_server.js b/src/server/system_services/bucket_server.js index 0dbaf92bd2..f4f994d73b 100644 --- a/src/server/system_services/bucket_server.js +++ b/src/server/system_services/bucket_server.js @@ -279,8 +279,8 @@ async function create_bucket(req) { }; // reorder read resources so that the write resource is the first in the list - const ordered_read_resources = write_resource ? - [write_resource].concat(read_resources.filter(rr => rr.resource !== write_resource.resource)) : read_resources; + const ordered_read_resources = write_resource ? [write_resource].concat( + read_resources.filter(rr => rr.resource !== write_resource.resource)) : read_resources; bucket.namespace = { read_resources: ordered_read_resources, @@ -593,6 +593,44 @@ async function delete_bucket_website(req) { }); } +/** + * + * CORS + * + */ +async function put_bucket_cors(req) { + dbg.log0('put_bucket_cors:', req.rpc_params); + const bucket = find_bucket(req); + await system_store.make_changes({ + update: { + buckets: [{ + _id: bucket._id, + cors_configuration_rules: req.rpc_params.cors_rules + }] + } + }); +} + +async function get_bucket_cors(req) { + dbg.log0('get_bucket_cors:', req.rpc_params); + const bucket = find_bucket(req, req.rpc_params.name); + return { + cors: bucket.cors_configuration_rules || [], + }; +} + +async function delete_bucket_cors(req) { + dbg.log0('delete_bucket_cors:', req.rpc_params); + const bucket = find_bucket(req, req.rpc_params.name); + await system_store.make_changes({ + update: { + buckets: [{ + _id: bucket._id, + $unset: { cors_configuration_rules: 1 } + }] + } + }); +} /** * @@ -639,6 +677,7 @@ async function read_bucket_sdk_info(req) { }) .then(get_bucket_info), notifications: bucket.notifications, + cors_configuration_rules: bucket.cors_configuration_rules, }; if (bucket.namespace) { @@ -2019,9 +2058,9 @@ async function validate_replication(req) { if (_.isEqual(db_rules._id, dst_bucket.replication_policy_id)) { const matching_rule = db_rules.rules.find( db_rule => - _.isEqual(src_bucket._id, db_rule.destination_bucket) && - (!db_rule.filter || db_rule.filter.prefix.toString().startsWith(prefix) || - prefix.toString().startsWith(db_rule.filter.prefix.toString())) + _.isEqual(src_bucket._id, db_rule.destination_bucket) && + (!db_rule.filter || db_rule.filter.prefix.toString().startsWith(prefix) || + prefix.toString().startsWith(db_rule.filter.prefix.toString())) ); if (matching_rule) { throw new RpcError('INVALID_REPLICATION_POLICY', @@ -2136,6 +2175,9 @@ exports.put_bucket_policy = put_bucket_policy; exports.get_bucket_policy = get_bucket_policy; exports.put_bucket_notification = put_bucket_notification; exports.get_bucket_notification = get_bucket_notification; +exports.put_bucket_cors = put_bucket_cors; +exports.get_bucket_cors = get_bucket_cors; +exports.delete_bucket_cors = delete_bucket_cors; exports.update_all_buckets_default_pool = update_all_buckets_default_pool; diff --git a/src/server/system_services/schemas/bucket_schema.js b/src/server/system_services/schemas/bucket_schema.js index 4b1c1168d1..05446ed08b 100644 --- a/src/server/system_services/schemas/bucket_schema.js +++ b/src/server/system_services/schemas/bucket_schema.js @@ -203,6 +203,10 @@ module.exports = { lifecycle_configuration_rules: { $ref: 'common_api#/definitions/bucket_lifecycle_configuration' }, + //cors rules if exist + cors_configuration_rules: { + $ref: 'common_api#/definitions/bucket_cors_configuration' + }, tagging: { $ref: 'common_api#/definitions/tagging', }, @@ -278,7 +282,7 @@ module.exports = { notifications: { type: 'array', items: { - $ref: 'common_api#/definitions/bucket_notification' + $ref: 'common_api#/definitions/bucket_notification' } }, } diff --git a/src/server/system_services/schemas/nsfs_bucket_schema.js b/src/server/system_services/schemas/nsfs_bucket_schema.js index 5a38368fa4..363f149004 100644 --- a/src/server/system_services/schemas/nsfs_bucket_schema.js +++ b/src/server/system_services/schemas/nsfs_bucket_schema.js @@ -81,8 +81,11 @@ module.exports = { notifications: { type: 'array', items: { - $ref: 'common_api#/definitions/bucket_notification' + $ref: 'common_api#/definitions/bucket_notification' } }, + cors_configuration_rules: { + $ref: 'common_api#/definitions/bucket_cors_configuration' + }, } }; diff --git a/src/test/system_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh b/src/test/system_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh index 65f6d93641..83053ca4ce 100644 --- a/src/test/system_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh +++ b/src/test/system_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh @@ -24,6 +24,7 @@ export CEPH_TEST_LOGS_DIR=/logs/ceph-nsfs-test-logs export CONFIG_DIR=/etc/noobaa.conf.d/ export FS_ROOT_1=/tmp/nsfs_root1/ export FS_ROOT_2=/tmp/nsfs_root2/ +export CONFIG_JS_allow_anonymous_access_in_test=true # Needed for allowing anon access for tests using ACL='public-read-write' # ==================================================================================== diff --git a/src/test/system_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh b/src/test/system_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh index bf1d752f01..fa53b427ca 100755 --- a/src/test/system_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh +++ b/src/test/system_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh @@ -32,6 +32,9 @@ export HOSTED_AGENTS_ADDR=wss://localhost:5446 export CEPH_TEST_LOGS_DIR=/logs/ceph-test-logs +export CONFIG_JS_OBJECT_SDK_BUCKET_CACHE_EXPIRY_MS=0 # Needed for disabling cache for ceph cors test and maybe some more +export CONFIG_JS_allow_anonymous_access_in_test=true # Needed for allowing anon access for tests using ACL='public-read-write' + # ==================================================================================== # Create the logs directory diff --git a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_black_list.txt b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_black_list.txt index 41959593eb..e098992817 100644 --- a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_black_list.txt +++ b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_black_list.txt @@ -207,10 +207,6 @@ s3tests_boto3/functional/test_s3.py::test_multipart_upload_size_too_small s3tests_boto3/functional/test_s3.py::test_abort_multipart_upload s3tests_boto3/functional/test_s3.py::test_multipart_copy_improper_range s3tests_boto3/functional/test_s3.py::test_100_continue -s3tests_boto3/functional/test_s3.py::test_set_cors -s3tests_boto3/functional/test_s3.py::test_cors_origin_wildcard -s3tests_boto3/functional/test_s3.py::test_cors_origin_response -s3tests_boto3/functional/test_s3.py::test_cors_header_option s3tests_boto3/functional/test_s3.py::test_set_tagging s3tests_boto3/functional/test_s3.py::test_multipart_resend_first_finishes_last s3tests_boto3/functional/test_s3.py::test_versioned_object_acl @@ -400,9 +396,7 @@ s3tests_boto3/functional/test_s3.py::test_sse_s3_encrypted_upload_1b s3tests_boto3/functional/test_s3.py::test_sse_s3_encrypted_upload_1kb s3tests_boto3/functional/test_s3.py::test_sse_s3_encrypted_upload_1mb s3tests_boto3/functional/test_s3.py::test_sse_s3_encrypted_upload_8mb -s3tests_boto3/functional/test_s3.py::test_cors_presigned_get_object s3tests_boto3/functional/test_s3.py::test_cors_presigned_get_object_tenant -s3tests_boto3/functional/test_s3.py::test_cors_presigned_put_object s3tests_boto3/functional/test_s3.py::test_cors_presigned_put_object_with_acl s3tests_boto3/functional/test_s3.py::test_cors_presigned_put_object_tenant s3tests_boto3/functional/test_s3.py::test_cors_presigned_put_object_tenant_with_acl diff --git a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_black_list.txt b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_black_list.txt index 849f5a712d..f0bdded771 100644 --- a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_black_list.txt +++ b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_black_list.txt @@ -130,6 +130,7 @@ s3tests_boto3/functional/test_s3.py::test_object_delete_key_bucket_gone s3tests_boto3/functional/test_s3.py::test_object_raw_get_object_gone s3tests_boto3/functional/test_s3.py::test_bucket_head_extended s3tests_boto3/functional/test_s3.py::test_object_raw_get_bucket_acl +s3tests_boto3/functional/test_s3.py::test_object_raw_get_object_acl s3tests_boto3/functional/test_s3.py::test_object_raw_response_headers s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_bucket_acl s3tests_boto3/functional/test_s3.py::test_object_raw_get_x_amz_expires_out_range_zero @@ -208,10 +209,6 @@ s3tests_boto3/functional/test_s3.py::test_multipart_upload_size_too_small s3tests_boto3/functional/test_s3.py::test_abort_multipart_upload s3tests_boto3/functional/test_s3.py::test_multipart_copy_improper_range s3tests_boto3/functional/test_s3.py::test_100_continue -s3tests_boto3/functional/test_s3.py::test_set_cors -s3tests_boto3/functional/test_s3.py::test_cors_origin_wildcard -s3tests_boto3/functional/test_s3.py::test_cors_origin_response -s3tests_boto3/functional/test_s3.py::test_cors_header_option s3tests_boto3/functional/test_s3.py::test_set_tagging s3tests_boto3/functional/test_s3.py::test_multipart_resend_first_finishes_last s3tests_boto3/functional/test_s3.py::test_versioned_object_acl @@ -352,4 +349,6 @@ s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_resourc s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_wrong_resource_tag_deny s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_resource_tag_princ_tag s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_resource_tag_copy_obj -s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_role_resource_tag \ No newline at end of file +s3tests_boto3/functional/test_sts.py::test_assume_role_with_web_identity_role_resource_tag +s3tests_boto3/functional/test_s3.py::test_cors_presigned_put_object_with_acl +s3tests_boto3/functional/test_s3.py::test_cors_presigned_put_object_tenant_with_acl \ No newline at end of file diff --git a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt index 5983b70872..3bb3389001 100644 --- a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt +++ b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt @@ -117,7 +117,6 @@ s3tests/functional/test_headers.py::test_bucket_create_bad_amz_date_before_epoch s3tests_boto3/functional/test_s3.py::test_post_object_wrong_bucket s3tests_boto3/functional/test_s3.py::test_object_raw_get_x_amz_expires_not_expired s3tests_boto3/functional/test_s3.py::test_object_raw_get_x_amz_expires_not_expired_tenant -s3tests_boto3/functional/test_s3.py::test_cors_presigned_get_object s3tests_boto3/functional/test_s3.py::test_cors_presigned_get_object_tenant s3tests_boto3/functional/test_s3.py::test_multipart_get_part s3tests_boto3/functional/test_s3.py::test_non_multipart_get_part @@ -133,10 +132,7 @@ s3tests/functional/test_headers.py::test_bucket_create_bad_date_none_aws2 s3tests_boto3/functional/test_s3.py::test_get_object_ifnonematch_good s3tests_boto3/functional/test_s3.py::test_get_object_ifmodifiedsince_failed s3tests_boto3/functional/test_s3.py::test_post_object_wrong_bucket -s3tests_boto3/functional/test_s3.py::test_cors_presigned_put_object -s3tests_boto3/functional/test_s3.py::test_cors_presigned_put_object_with_acl s3tests_boto3/functional/test_s3.py::test_cors_presigned_put_object_tenant -s3tests_boto3/functional/test_s3.py::test_cors_presigned_put_object_tenant_with_acl s3tests_boto3/functional/test_s3.py::test_object_presigned_put_object_with_acl_tenant s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_newer_noncurrent s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_size_gt diff --git a/src/util/http_utils.js b/src/util/http_utils.js index 279ef780db..fc142d9700 100644 --- a/src/util/http_utils.js +++ b/src/util/http_utils.js @@ -622,10 +622,11 @@ function set_amz_headers(req, res) { /** * @typedef {{ * allow_origin: string; - * allow_credentials: string; * allow_methods: string; - * allow_headers: string; - * expose_headers: string; + * allow_headers?: string; + * expose_headers?: string; + * allow_credentials?: string; + * max_age?: number; * }} CORSConfig * @param {http.IncomingMessage} req * @param {http.ServerResponse} res @@ -633,24 +634,51 @@ function set_amz_headers(req, res) { */ function set_cors_headers(req, res, cors) { res.setHeader('Access-Control-Allow-Origin', cors.allow_origin); - res.setHeader('Access-Control-Allow-Credentials', cors.allow_credentials); res.setHeader('Access-Control-Allow-Methods', cors.allow_methods); - res.setHeader('Access-Control-Allow-Headers', cors.allow_headers); - res.setHeader('Access-Control-Expose-Headers', cors.expose_headers); + if (cors.allow_headers) res.setHeader('Access-Control-Allow-Headers', cors.allow_headers); + if (cors.expose_headers) res.setHeader('Access-Control-Expose-Headers', cors.expose_headers); + if (cors.allow_credentials) res.setHeader('Access-Control-Allow-Credentials', cors.allow_credentials); + if (cors.max_age) res.setHeader('Access-Control-Max-Age', cors.max_age); } /** + * * @typedef {{ + * allowed_origins: string[]; + * allowed_credentials: string[]; + * allowed_methods: string[]; + * allowed_headers: string[]; + * expose_headers: string[]; + * max_age: number; + * }} CORSRule * @param {http.IncomingMessage} req * @param {http.ServerResponse} res + * @param {CORSRule[]} cors_rules */ -function set_cors_headers_s3(req, res) { - if (config.S3_CORS_ENABLED) { +function set_cors_headers_s3(req, res, cors_rules) { + if (!config.S3_CORS_ENABLED || !cors_rules) return; + + // based on https://docs.aws.amazon.com/AmazonS3/latest/userguide/cors.html + const match_method = req.headers['access-control-request-method'] || req.method; + const match_origin = req.headers.origin; + const match_header = req.headers['access-control-request-headers']; // not a must + const matched_rule = req.headers.origin && ( // find the first rule with origin and method match + cors_rules.find(rule => { + const allowed_origins_regex = rule.allowed_origins.map(r => RegExp(`^${r.replace(/\*/g, '.*')}$`)); + const allowed_headers_regex = rule.allowed_headers?.map(r => RegExp(`^${r.replace(/\*/g, '.*')}$`)); + return allowed_origins_regex.some(r => r.test(match_origin)) && + rule.allowed_methods.includes(match_method) && + // we can match if no request headers or if reuqest headers match the rule allowed headers + (!match_header || allowed_headers_regex?.some(r => r.test(match_header))); + })); + if (matched_rule) { + // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html set_cors_headers(req, res, { - allow_origin: config.S3_CORS_ALLOW_ORIGIN, - allow_credentials: config.S3_CORS_ALLOW_CREDENTIAL, - allow_methods: config.S3_CORS_ALLOW_METHODS, - allow_headers: config.S3_CORS_ALLOW_HEADERS, - expose_headers: config.S3_CORS_EXPOSE_HEADERS, + allow_origin: matched_rule.allowed_origins.includes('*') ? '*' : req.headers.origin, + allow_methods: matched_rule.allowed_methods.join(','), + allow_headers: matched_rule.allowed_headers?.join(','), + expose_headers: matched_rule.expose_headers?.join(','), + allow_credentials: 'true', + max_age: matched_rule?.max_age }); } }