From 87e5d6808ca2d302b761014784f7c158c4cc2cc4 Mon Sep 17 00:00:00 2001 From: Paris Kasidiaris Date: Mon, 4 Mar 2019 15:30:58 +0200 Subject: [PATCH 01/17] Start migrating code from old branch. --- README.md | 4 +- ceryx/Dockerfile | 5 +- ceryx/nginx/conf/ceryx.conf.tmpl | 14 +---- ceryx/nginx/conf/nginx.conf.tmpl | 2 +- ceryx/nginx/lualib/certificate.lua | 60 +++++++++++++++++++ .../lualib/{https.lua => letsencrypt.lua} | 0 6 files changed, 69 insertions(+), 16 deletions(-) create mode 100644 ceryx/nginx/lualib/certificate.lua rename ceryx/nginx/lualib/{https.lua => letsencrypt.lua} (100%) diff --git a/README.md b/README.md index a778ac4..d8591a3 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,8 @@ Ceryx is configured with the following environment variables: - `CERYX_REDIS_PASSWORD`: Optional password to use for authenticating with Redis (default: None) - `CERYX_REDIS_PORT`: The where Redis should be reached (default: `6379`) - `CERYX_REDIS_PREFIX`: The prefix to use in Ceryx-related Redis keys (default: `ceryx`) - - `CERYX_SSL_CERT_KEY`: The path to the fallback SSL certificate key (default: randomly generated) - - `CERYX_SSL_CERT`: The path to the fallback SSL certificate (default: randomly generated) + - `CERYX_SSL_DEFAULT_CERTIFICATE`: The path to the fallback SSL certificate (default: `/etc/ceryx/ssl/default.crt` — randomly generated at build time) + - `CERYX_SSL_DEFAULT_CERTIFICATE_KEY`: The path to the fallback SSL certificate key (default: `/etc/ceryx/ssl/default.key` — randomly generated at build time) ## Adjusting log level diff --git a/ceryx/Dockerfile b/ceryx/Dockerfile index 59fbb51..13d4cdd 100644 --- a/ceryx/Dockerfile +++ b/ceryx/Dockerfile @@ -4,10 +4,11 @@ ARG user=www-data ARG group=www-data RUN mkdir -p /etc/letsencrypt &&\ + mkdir -p /etc/ceryx/ssl &&\ openssl req -new -newkey rsa:2048 -days 3650 -nodes -x509 \ -subj '/CN=sni-support-required-for-valid-ssl' \ - -keyout /etc/ssl/resty-auto-ssl-fallback.key \ - -out /etc/ssl/resty-auto-ssl-fallback.crt + -keyout /etc/ceryx/ssl/default.key \ + -out /etc/ceryx/ssl/default.crt # Install dockerize binary, for templated configs # https://github.com/jwilder/dockerize diff --git a/ceryx/nginx/conf/ceryx.conf.tmpl b/ceryx/nginx/conf/ceryx.conf.tmpl index 626b5e6..3b1af91 100644 --- a/ceryx/nginx/conf/ceryx.conf.tmpl +++ b/ceryx/nginx/conf/ceryx.conf.tmpl @@ -14,17 +14,9 @@ server { listen 443 ssl; default_type text/html; - ssl_certificate {{ default .Env.CERYX_SSL_CERT "/etc/ssl/resty-auto-ssl-fallback.crt" }}; - ssl_certificate_key {{ default .Env.CERYX_SSL_CERT_KEY "/etc/ssl/resty-auto-ssl-fallback.key" }}; - - {{ if ne (lower (default .Env.CERYX_DISABLE_LETS_ENCRYPT "")) "true" }} - # Generate Let's Encrypt certificates automatically, if - # `CERYX_DISABLE_LETS_ENCRYPT` is *not* set to `true`. - - ssl_certificate_by_lua_block { - auto_ssl:ssl_certificate() - } - {{ end }} + ssl_certificate {{ default .Env.CERYX_SSL_DEFAULT_CERTIFICATE "/etc/ceryx/ssl/default.crt" }}; + ssl_certificate_key {{ default .Env.CERYX_SSL_DEFAULT_CERTIFICATE_KEY "/etc/ceryx/ssl/default.key" }}; + ssl_certificate_by_lua_file lualib/certificate.lua; location /.well-known/acme-challenge { content_by_lua_block { diff --git a/ceryx/nginx/conf/nginx.conf.tmpl b/ceryx/nginx/conf/nginx.conf.tmpl index b64593d..4b6164d 100644 --- a/ceryx/nginx/conf/nginx.conf.tmpl +++ b/ceryx/nginx/conf/nginx.conf.tmpl @@ -42,7 +42,7 @@ http { # Enable automatic Let's Encryps certificate generation, if # `CERYX_DISABLE_LETS_ENCRYPT` is *not* set to `true`. # Check out https://github.com/openresty/lua-resty-core - init_by_lua_file "lualib/https.lua"; + init_by_lua_file "lualib/letsencrypt.lua"; init_worker_by_lua_block { auto_ssl:init_worker() diff --git a/ceryx/nginx/lualib/certificate.lua b/ceryx/nginx/lualib/certificate.lua new file mode 100644 index 0000000..485f376 --- /dev/null +++ b/ceryx/nginx/lualib/certificate.lua @@ -0,0 +1,60 @@ +local ssl = require "ngx.ssl" +local redis = require "ceryx.redis" + +local domain, err = ssl.server_name() +local redisClient = redis:client() + +ngx.log(ngx.STDERR, "Searching for SSL Certificate for " .. domain) + +local certificate_key = redis.prefix .. ":certificates:" .. domain +local certificate = redisClient:hgetall(certificate_key) + +-- If Redis did not return an empty table, run the following code +if next(certificate) ~= nil then + ngx.log(ngx.STDERR, "Found SSL Certificate for " .. domain .. " in Redis. Using this.") + + local path_to_bundle = certificate[2] + local path_to_key = certificate[4] + + ngx.log(ngx.STDERR, path_to_bundle) + ngx.log(ngx.STDERR, path_to_key) + + local bundle_file = assert(io.open(path_to_bundle, "r")) + local bundle_data = bundle_file:read("*all") + bundle_file:close() + + local key_file = assert(io.open(path_to_key, "r")) + local key_data = key_file:read("*all") + key_file:close() + + -- Convert data from PEM to DER + local bundle_der, bundle_der_err = ssl.cert_pem_to_der(bundle_data) + if not bundle_der or bundle_der_err then + ngx.log(ngx.ERROR, "Could not convert PEM bundle to DER. Error: " .. (bundle_der_err or "")) + ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) + end + + local key_der, key_der_err = ssl.priv_key_pem_to_der(key_data) + if not key_der or key_der_err then + ngx.log(ngx.ERROR, "Could not convert PEM key to DER. Error: " .. (key_der_err or "")) + ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) + end + + -- Set the certificate information for the current SSL Session + local session_bundle_ok, session_bundle_err = ssl.set_der_cert(bundle_der) + if not session_bundle_ok then + ngx.log(ngx.ERROR, "Could not set the certificate for the current SSL Session. Error: " .. (session_bundle_err or "")) + ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) + end + + local session_key_ok, session_key_err = ssl.set_der_priv_key(key_der) + if not session_key_ok then + ngx.log(ngx.ERROR, "Could not set the key for the current SSL Session. Error: " .. (session_key_err or "")) + ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) + end +else + ngx.log(ngx.ERROR, "Did not find SSL Certificate for " .. domain .. " in Redis. Using to automatic Let's Encrypt.") + auto_ssl:ssl_certificate() +end + +ngx.log(ngx.DEBUG, "Completed SSL negotiation for " .. domain) \ No newline at end of file diff --git a/ceryx/nginx/lualib/https.lua b/ceryx/nginx/lualib/letsencrypt.lua similarity index 100% rename from ceryx/nginx/lualib/https.lua rename to ceryx/nginx/lualib/letsencrypt.lua From 14bf2681c3e87e169e93cc1061dad785d3af689f Mon Sep 17 00:00:00 2001 From: Paris Kasidiaris Date: Mon, 4 Mar 2019 16:22:55 +0200 Subject: [PATCH 02/17] Complete first major SSL refactoring. --- README.md | 2 +- ceryx/nginx/conf/ceryx.conf.tmpl | 2 +- ceryx/nginx/lualib/certificate.lua | 75 +++++++++++------------ ceryx/nginx/lualib/ceryx/certificates.lua | 44 +++++++++++++ ceryx/nginx/lualib/ceryx/utils.lua | 8 +++ docker-compose.yml | 2 + 6 files changed, 91 insertions(+), 42 deletions(-) create mode 100644 ceryx/nginx/lualib/ceryx/certificates.lua diff --git a/README.md b/README.md index d8591a3..def7bff 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Ceryx is configured with the following environment variables: - `CERYX_REDIS_PORT`: The where Redis should be reached (default: `6379`) - `CERYX_REDIS_PREFIX`: The prefix to use in Ceryx-related Redis keys (default: `ceryx`) - `CERYX_SSL_DEFAULT_CERTIFICATE`: The path to the fallback SSL certificate (default: `/etc/ceryx/ssl/default.crt` — randomly generated at build time) - - `CERYX_SSL_DEFAULT_CERTIFICATE_KEY`: The path to the fallback SSL certificate key (default: `/etc/ceryx/ssl/default.key` — randomly generated at build time) + - `CERYX_SSL_DEFAULT_KEY`: The path to the fallback SSL certificate key (default: `/etc/ceryx/ssl/default.key` — randomly generated at build time) ## Adjusting log level diff --git a/ceryx/nginx/conf/ceryx.conf.tmpl b/ceryx/nginx/conf/ceryx.conf.tmpl index 3b1af91..92e13c5 100644 --- a/ceryx/nginx/conf/ceryx.conf.tmpl +++ b/ceryx/nginx/conf/ceryx.conf.tmpl @@ -15,7 +15,7 @@ server { default_type text/html; ssl_certificate {{ default .Env.CERYX_SSL_DEFAULT_CERTIFICATE "/etc/ceryx/ssl/default.crt" }}; - ssl_certificate_key {{ default .Env.CERYX_SSL_DEFAULT_CERTIFICATE_KEY "/etc/ceryx/ssl/default.key" }}; + ssl_certificate_key {{ default .Env.CERYX_SSL_DEFAULT_KEY "/etc/ceryx/ssl/default.key" }}; ssl_certificate_by_lua_file lualib/certificate.lua; location /.well-known/acme-challenge { diff --git a/ceryx/nginx/lualib/certificate.lua b/ceryx/nginx/lualib/certificate.lua index 485f376..b5ac6a4 100644 --- a/ceryx/nginx/lualib/certificate.lua +++ b/ceryx/nginx/lualib/certificate.lua @@ -1,60 +1,55 @@ +local certificates = require "ceryx.certificates" local ssl = require "ngx.ssl" local redis = require "ceryx.redis" +local utils = require "ceryx.utils" -local domain, err = ssl.server_name() -local redisClient = redis:client() +local disable_lets_encrypt = utils.getenv("CERYX_DISABLE_LETS_ENCRYPT", ""):lower() == "true" +local host, host_err = ssl.server_name() -ngx.log(ngx.STDERR, "Searching for SSL Certificate for " .. domain) - -local certificate_key = redis.prefix .. ":certificates:" .. domain -local certificate = redisClient:hgetall(certificate_key) - --- If Redis did not return an empty table, run the following code -if next(certificate) ~= nil then - ngx.log(ngx.STDERR, "Found SSL Certificate for " .. domain .. " in Redis. Using this.") - - local path_to_bundle = certificate[2] - local path_to_key = certificate[4] - - ngx.log(ngx.STDERR, path_to_bundle) - ngx.log(ngx.STDERR, path_to_key) - - local bundle_file = assert(io.open(path_to_bundle, "r")) - local bundle_data = bundle_file:read("*all") - bundle_file:close() +if not host then + ngx.log(ngx.ERROR, "Could not retrieve SSL Server Name: " .. host_err) + return ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) +end - local key_file = assert(io.open(path_to_key, "r")) - local key_data = key_file:read("*all") - key_file:close() +local host_certificates = certificates.getCertificatesForHost(host) +if certificates ~= nil then -- Convert data from PEM to DER - local bundle_der, bundle_der_err = ssl.cert_pem_to_der(bundle_data) - if not bundle_der or bundle_der_err then - ngx.log(ngx.ERROR, "Could not convert PEM bundle to DER. Error: " .. (bundle_der_err or "")) - ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) + local certificate_der, certificate_der_err = ssl.cert_pem_to_der(host_certificates["certificate"]) + if not certificate_der or certificate_der_err then + ngx.log(ngx.ERROR, "Could not convert SSL Certificate to DER. Error: " .. (certificate_der_err or "")) + ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) end local key_der, key_der_err = ssl.priv_key_pem_to_der(key_data) if not key_der or key_der_err then - ngx.log(ngx.ERROR, "Could not convert PEM key to DER. Error: " .. (key_der_err or "")) - ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) + ngx.log(ngx.ERROR, "Could not convert PEM key to DER. Error: " .. (key_der_err or "")) + ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) end -- Set the certificate information for the current SSL Session - local session_bundle_ok, session_bundle_err = ssl.set_der_cert(bundle_der) - if not session_bundle_ok then - ngx.log(ngx.ERROR, "Could not set the certificate for the current SSL Session. Error: " .. (session_bundle_err or "")) - ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) + local ssl_certificate_ok, ssl_certficate_error = ssl.set_der_cert(certificate_der) + + if not ssl_certificate_ok then + ngx.log( + ngx.ERROR, + "Could not set the certificate for the current SSL Session. Error: " .. (ssl_certficate_error or "") + ) + ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) end - local session_key_ok, session_key_err = ssl.set_der_priv_key(key_der) - if not session_key_ok then - ngx.log(ngx.ERROR, "Could not set the key for the current SSL Session. Error: " .. (session_key_err or "")) - ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) + local ssl_key_ok, ssl_key_error = ssl.set_der_priv_key(key_der) + if not ssl_key_ok then + ngx.log(ngx.ERROR, "Could not set the key for the current SSL Session. Error: " .. (ssl_key_error or "")) + ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) end else - ngx.log(ngx.ERROR, "Did not find SSL Certificate for " .. domain .. " in Redis. Using to automatic Let's Encrypt.") - auto_ssl:ssl_certificate() + ngx.log(ngx.INFO, "No valid SSL certificate has been configured for " .. host .. ".") + + if not disable_lets_encrypt then + ngx.log(ngx.INFO, "Passing SSL certificate handling for " .. host .. " to Let's Encrypt.") + auto_ssl:ssl_certificate() + end end -ngx.log(ngx.DEBUG, "Completed SSL negotiation for " .. domain) \ No newline at end of file +ngx.log(ngx.DEBUG, "Completed SSL negotiation for " .. host) diff --git a/ceryx/nginx/lualib/ceryx/certificates.lua b/ceryx/nginx/lualib/ceryx/certificates.lua new file mode 100644 index 0000000..c32320d --- /dev/null +++ b/ceryx/nginx/lualib/ceryx/certificates.lua @@ -0,0 +1,44 @@ +local redis = require "ceryx.redis" +local ssl = require "ngx.ssl" + +local exports = {} + +function getRedisKeyForHost(host) + return redis.prefix .. ":certificates:" .. host +end + +function getCertificatesForHost(host) + ngx.log(ngx.DEBUG, "Looking for SSL sertificate for " .. host) + + local certificates_redis_key = getRedisKeyForHost(host) + local certificate_path, certificate_err = redisClient:hget(certificates_redis_key, "certificate_path") + local key_path, key_err = redisClient:hget(certificates_redis_key, "key_path") + + if certificate == nil then + ngx.log( + ngx.ERROR, + "Could not retrieve SSL certificate path for " .. host .. " from Redis: " .. (certificate_err or "N/A") + ) + return nil + end + + if key_path == nil then + ngx.log(ngx.ERROR, "Could not retrieve SSL key path for " .. host .. " from Redis: " .. (key_err or "N/A")) + return nil + end + + ngx.log(ngx.DEBUG, "Found SSL certificates for " .. host .. " in Redis.") + + local certificate_data = utils.read_file(certificate_path) + local key_data = utils.read_file(key_path) + local data = {} + + data["certificate"] = certificate_data + data["key"] = key_data + + return data +end + +exports.getCertificatesForHost = getCertificatesForHost + +return exports diff --git a/ceryx/nginx/lualib/ceryx/utils.lua b/ceryx/nginx/lualib/ceryx/utils.lua index b1b1b9f..0b86688 100644 --- a/ceryx/nginx/lualib/ceryx/utils.lua +++ b/ceryx/nginx/lualib/ceryx/utils.lua @@ -42,4 +42,12 @@ function exports.getenv(variable, default) return default end +function exports.read_file(path) + local file = assert(io.open(path, "r")) + local file_data = file:read("*all") + file:close() + + return file_data +end + return exports diff --git a/docker-compose.yml b/docker-compose.yml index 940cb61..8767606 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,8 @@ services: - redis environment: CERYX_DISABLE_LETS_ENCRYPT: ${CERYX_DISABLE_LETS_ENCRYPT:-false} + CERYX_SSL_DEFAULT_CERTIFICATE: ${CERYX_SSL_DEFAULT_CERTIFICATE:-/etc/ceryx/ssl/default.crt} + CERYX_SSL_DEFAULT_KEY: ${CERYX_SSL_DEFAULT_KEY:-/etc/ceryx/ssl/default.key} CERYX_DOCKERIZE_EXTRA_ARGS: -no-overwrite CERYX_REDIS_HOST: ${CERYX_REDIS_HOST:-redis} CERYX_REDIS_PORT: ${CERYX_REDIS_PORT:-6379} From 463b69c6dd96afa1ad0266442e57873e28b05ed4 Mon Sep 17 00:00:00 2001 From: Paris Kasidiaris Date: Wed, 24 Apr 2019 09:56:27 +0300 Subject: [PATCH 03/17] Remove unused `redis.lua` --- ceryx/redis.lua | 36 ------------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 ceryx/redis.lua diff --git a/ceryx/redis.lua b/ceryx/redis.lua deleted file mode 100644 index 883805e..0000000 --- a/ceryx/redis.lua +++ /dev/null @@ -1,36 +0,0 @@ -local redis = require "resty.redis" - -function client() - local prefix = os.getenv("CERYX_REDIS_PREFIX") - if not prefix then prefix = "ceryx" end - - -- Prepare the Redis client - ngx.log(ngx.DEBUG, "Preparing Redis client.") - local red = redis:new() - red:set_timeout(100) -- 100 ms - local redis_host = os.getenv("CERYX_REDIS_HOST") - if not redis_host then redis_host = "127.0.0.1" end - local redis_port = os.getenv("CERYX_REDIS_PORT") - if not redis_port then redis_port = 6379 end - local redis_password = os.getenv("CERYX_REDIS_PASSWORD") - if not redis_password then redis_password = nil end - local res, err = red:connect(redis_host, redis_port) - - -- Return if could not connect to Redis - if not res then - ngx.log(ngx.DEBUG, "Could not prepare Redis client: " .. err) - return ngx.exit(ngx.HTTP_SERVER_ERROR) - end - - ngx.log(ngx.DEBUG, "Redis client prepared.") - - if redis_password then - ngx.log(ngx.DEBUG, "Authenticating with Redis.") - local res, err = red:auth(redis_password) - if not res then - ngx.ERR("Could not authenticate with Redis: ", err) - return ngx.exit(ngx.HTTP_SERVER_ERROR) - end - end - ngx.log(ngx.DEBUG, "Authenticated with Redis.") -end \ No newline at end of file From c6aac47b0bb12012faf469af8370a356b452992c Mon Sep 17 00:00:00 2001 From: Paris Kasidiaris Date: Wed, 24 Apr 2019 12:25:51 +0300 Subject: [PATCH 04/17] Implement custom SSL keys on a per-route basis. --- ceryx/nginx/lualib/certificate.lua | 2 +- ceryx/nginx/lualib/ceryx/certificates.lua | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ceryx/nginx/lualib/certificate.lua b/ceryx/nginx/lualib/certificate.lua index b5ac6a4..7c34980 100644 --- a/ceryx/nginx/lualib/certificate.lua +++ b/ceryx/nginx/lualib/certificate.lua @@ -21,7 +21,7 @@ if certificates ~= nil then ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) end - local key_der, key_der_err = ssl.priv_key_pem_to_der(key_data) + local key_der, key_der_err = ssl.priv_key_pem_to_der(host_certificates["key"]) if not key_der or key_der_err then ngx.log(ngx.ERROR, "Could not convert PEM key to DER. Error: " .. (key_der_err or "")) ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) diff --git a/ceryx/nginx/lualib/ceryx/certificates.lua b/ceryx/nginx/lualib/ceryx/certificates.lua index c32320d..bbab0eb 100644 --- a/ceryx/nginx/lualib/ceryx/certificates.lua +++ b/ceryx/nginx/lualib/ceryx/certificates.lua @@ -1,5 +1,6 @@ local redis = require "ceryx.redis" local ssl = require "ngx.ssl" +local utils = require "ceryx.utils" local exports = {} @@ -9,28 +10,29 @@ end function getCertificatesForHost(host) ngx.log(ngx.DEBUG, "Looking for SSL sertificate for " .. host) - + local redisClient = redis:client() local certificates_redis_key = getRedisKeyForHost(host) local certificate_path, certificate_err = redisClient:hget(certificates_redis_key, "certificate_path") local key_path, key_err = redisClient:hget(certificates_redis_key, "key_path") - if certificate == nil then - ngx.log( - ngx.ERROR, - "Could not retrieve SSL certificate path for " .. host .. " from Redis: " .. (certificate_err or "N/A") - ) + if certificate_path == nil then + ngx.log(ngx.ERR, "Could not retrieve SSL certificate path for " .. host .. " from Redis: " .. (certificate_err or "N/A")) return nil end if key_path == nil then - ngx.log(ngx.ERROR, "Could not retrieve SSL key path for " .. host .. " from Redis: " .. (key_err or "N/A")) + ngx.log(ngx.ERR, "Could not retrieve SSL key path for " .. host .. " from Redis: " .. (key_err or "N/A")) return nil end ngx.log(ngx.DEBUG, "Found SSL certificates for " .. host .. " in Redis.") + ngx.log(ngx.DEBUG, "Reading certificate file for " .. host .. ": " .. certificate_path) local certificate_data = utils.read_file(certificate_path) + + ngx.log(ngx.DEBUG, "Reading key file for " .. host .. ": " .. key_path) local key_data = utils.read_file(key_path) + local data = {} data["certificate"] = certificate_data From f97d7b03d7025665eb264bce2fdf15bf5876612e Mon Sep 17 00:00:00 2001 From: Paris Kasidiaris Date: Wed, 24 Apr 2019 12:39:12 +0300 Subject: [PATCH 05/17] Move certificates into settings. --- ceryx/nginx/lualib/ceryx/certificates.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ceryx/nginx/lualib/ceryx/certificates.lua b/ceryx/nginx/lualib/ceryx/certificates.lua index bbab0eb..104e858 100644 --- a/ceryx/nginx/lualib/ceryx/certificates.lua +++ b/ceryx/nginx/lualib/ceryx/certificates.lua @@ -5,7 +5,7 @@ local utils = require "ceryx.utils" local exports = {} function getRedisKeyForHost(host) - return redis.prefix .. ":certificates:" .. host + return redis.prefix .. ":settings:" .. host end function getCertificatesForHost(host) From add73850479c5f94421764f35f74fabc5fa547af Mon Sep 17 00:00:00 2001 From: Paris Kasidiaris Date: Wed, 24 Apr 2019 18:38:45 +0300 Subject: [PATCH 06/17] Tests werk. --- bin/test | 8 ++ ceryx/Pipfile | 2 + ceryx/Pipfile.lock | 41 +++++---- ceryx/nginx/lualib/ceryx/certificates.lua | 3 - ceryx/tests/client.py | 103 ++++++++++++++++++++++ ceryx/tests/test_certificates.py | 55 ++++++++++++ docker-compose.override.yml | 2 +- docker-compose.test.yml | 10 ++- docker-compose.yml | 2 +- 9 files changed, 204 insertions(+), 22 deletions(-) create mode 100755 bin/test create mode 100644 ceryx/tests/client.py create mode 100644 ceryx/tests/test_certificates.py diff --git a/bin/test b/bin/test new file mode 100755 index 0000000..e0de82d --- /dev/null +++ b/bin/test @@ -0,0 +1,8 @@ +#! /bin/bash + +set -ex + +export DOCKER_COMPOSE_VERSION=1.23.2 +export COMPOSE_FILE=docker-compose.yml:docker-compose.override.yml:docker-compose.test.yml + +docker-compose run test \ No newline at end of file diff --git a/ceryx/Pipfile b/ceryx/Pipfile index 12b317e..85eb04f 100644 --- a/ceryx/Pipfile +++ b/ceryx/Pipfile @@ -9,6 +9,8 @@ name = "pypi" pytest = "*" requests = "*" black = "==18.9b0" +redis = "*" +urllib3 = "*" [requires] python_version = "3.6" diff --git a/ceryx/Pipfile.lock b/ceryx/Pipfile.lock index 2a96d81..1a4703a 100644 --- a/ceryx/Pipfile.lock +++ b/ceryx/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "9b114b9d5064e0882163dd1bcf406f3b55526a07687e182c3f095301d54a17f1" + "sha256": "d5e51dbb4c258b0dde740fec8a0e1eff435ac14b3c808cb0ff2a900ed1eefa38" }, "pipfile-spec": 6, "requires": { @@ -33,10 +33,10 @@ }, "attrs": { "hashes": [ - "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", - "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" ], - "version": "==18.2.0" + "version": "==19.1.0" }, "black": { "hashes": [ @@ -48,10 +48,10 @@ }, "certifi": { "hashes": [ - "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", - "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", + "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" ], - "version": "==2018.11.29" + "version": "==2019.3.9" }, "chardet": { "hashes": [ @@ -76,11 +76,11 @@ }, "more-itertools": { "hashes": [ - "sha256:0125e8f60e9e031347105eb1682cef932f5e97d7b9a1a28d9bf00c22a5daef40", - "sha256:590044e3942351a1bdb1de960b739ff4ce277960f2425ad4509446dbace8d9d1" + "sha256:2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7", + "sha256:c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a" ], "markers": "python_version > '2.7'", - "version": "==6.0.0" + "version": "==7.0.0" }, "pluggy": { "hashes": [ @@ -98,11 +98,19 @@ }, "pytest": { "hashes": [ - "sha256:067a1d4bf827ffdd56ad21bd46674703fce77c5957f6c1eef731f6146bfcef1c", - "sha256:9687049d53695ad45cf5fdc7bbd51f0c49f1ea3ecfc4b7f3fde7501b541f17f4" + "sha256:3773f4c235918987d51daf1db66d51c99fac654c81d6f2f709a046ab446d5e5d", + "sha256:b7802283b70ca24d7119b32915efa7c409982f59913c1a6c0640aacf118b95f5" ], "index": "pypi", - "version": "==4.3.0" + "version": "==4.4.1" + }, + "redis": { + "hashes": [ + "sha256:6946b5dca72e86103edc8033019cc3814c031232d339d5f4533b02ea85685175", + "sha256:8ca418d2ddca1b1a850afa1680a7d2fd1f3322739271de4b704e0d4668449273" + ], + "index": "pypi", + "version": "==3.2.1" }, "requests": { "hashes": [ @@ -128,10 +136,11 @@ }, "urllib3": { "hashes": [ - "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", - "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", + "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3" ], - "version": "==1.24.1" + "index": "pypi", + "version": "==1.24.2" } } } diff --git a/ceryx/nginx/lualib/ceryx/certificates.lua b/ceryx/nginx/lualib/ceryx/certificates.lua index 104e858..0e14895 100644 --- a/ceryx/nginx/lualib/ceryx/certificates.lua +++ b/ceryx/nginx/lualib/ceryx/certificates.lua @@ -27,10 +27,7 @@ function getCertificatesForHost(host) ngx.log(ngx.DEBUG, "Found SSL certificates for " .. host .. " in Redis.") - ngx.log(ngx.DEBUG, "Reading certificate file for " .. host .. ": " .. certificate_path) local certificate_data = utils.read_file(certificate_path) - - ngx.log(ngx.DEBUG, "Reading key file for " .. host .. ": " .. key_path) local key_data = utils.read_file(key_path) local data = {} diff --git a/ceryx/tests/client.py b/ceryx/tests/client.py new file mode 100644 index 0000000..9234146 --- /dev/null +++ b/ceryx/tests/client.py @@ -0,0 +1,103 @@ +import os +import socket + +from requests import Session +from requests.adapters import DEFAULT_POOLBLOCK, HTTPAdapter +from urllib3.connection import HTTPConnection, HTTPSConnection +from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool +from urllib3.poolmanager import PoolManager + + +DEFAULT_CERYX_HOST = "ceryx" # Set by Docker Compose in tests +CERYX_HOST = os.getenv("CERYX_HOST", DEFAULT_CERYX_HOST) + + +class CeryxTestsHTTPConnection(HTTPConnection): + """ + Custom-built HTTPConnection for Ceryx tests. + """ + + @property + def host(self): + return self._dns_host.rstrip('.') + + @host.setter + def host(self, value): + """ + Do exactly what the parent class does. + """ + self._dns_host = CERYX_HOST if value.endswith(".ceryx.test") else value + + +class CeryxTestsHTTPSConnection(CeryxTestsHTTPConnection, HTTPSConnection): + def __init__( + self, host, port=None, key_file=None, cert_file=None, + key_password=None, strict=None, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT, ssl_context=None, + server_hostname=None, **kw, + ): + + CeryxTestsHTTPConnection.__init__( + self, host, port, strict=strict, timeout=timeout, **kw, + ) + + self.key_file = key_file + self.cert_file = cert_file + self.key_password = key_password + self.ssl_context = ssl_context + self.server_hostname = server_hostname + + # Required property for Google AppEngine 1.9.0 which otherwise causes + # HTTPS requests to go out as HTTP. (See Issue #356) + self._protocol = 'https' + + +class CeryxTestsHTTPConnectionPool(HTTPConnectionPool): + ConnectionCls = CeryxTestsHTTPConnection + + def __init__(self, host, *args, **kwargs): + self._impostor_host = host + super().__init__(host, *args, **kwargs) + + def urlopen(self, *args, **kwargs): + kwargs["headers"]["Host"] = self._impostor_host + return super().urlopen(*args, **kwargs) + + +class CeryxTestsHTTPSConnectionPool(HTTPSConnectionPool): + ConnectionCls = CeryxTestsHTTPSConnection + + def __init__(self, host, *args, **kwargs): + super().__init__(host, *args, **kwargs) + self.conn_kw["server_hostname"] = host + + +class CeryxTestsPoolManager(PoolManager): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.pool_classes_by_scheme = { + "http": CeryxTestsHTTPConnectionPool, + "https": CeryxTestsHTTPSConnectionPool, + } + + +class CeryxTestsHTTPAdapter(HTTPAdapter): + def init_poolmanager( + self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs, + ): + # Comment from original Requests HTTPAdapter: Save these values for pickling + self._pool_connections = connections + self._pool_maxsize = maxsize + self._pool_block = block + + self.poolmanager = CeryxTestsPoolManager( + num_pools=connections, maxsize=maxsize, block=block, strict=True, + **pool_kwargs, + ) + + +class Client(Session): + def __init__(self): + super().__init__() + self.mount("http://", CeryxTestsHTTPAdapter()) + self.mount("https://", CeryxTestsHTTPAdapter()) diff --git a/ceryx/tests/test_certificates.py b/ceryx/tests/test_certificates.py new file mode 100644 index 0000000..53fb132 --- /dev/null +++ b/ceryx/tests/test_certificates.py @@ -0,0 +1,55 @@ +import os +import stat +import subprocess +import uuid + +import redis +import requests + +import client + +ALL_CAN_READ = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH + +CERTIFICATE_ROOT = "/usr/local/share/certificates" +CERYX_HOST = "ceryx" + +redis_client = redis.Redis(host='redis') +testing_client = client.Client() + + +def test_custom_certificate(): + """ + Ensure that Ceryx uses the given certificate for each route, if configured + so. + """ + certificate_id = uuid.uuid4() + certificate_path = f"{CERTIFICATE_ROOT}/{certificate_id}.crt" + key_path = f"{CERTIFICATE_ROOT}/{certificate_id}.key" + hostname = "custom-certificate.ceryx.test" + + command = [ + "openssl", + "req", "-x509", + "-newkey", "rsa:4096", + "-keyout", key_path, + "-out", certificate_path, + "-days", "1", + "-subj", f"/C=GR/ST=Attica/L=Athens/O=SourceLair/OU=Org/CN={hostname}", + "-nodes", + ] + subprocess.run( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, + ) + os.chmod(certificate_path, ALL_CAN_READ) + os.chmod(key_path, ALL_CAN_READ) + + api_base_url = "http://api:5555/" + + route_redis_key = f"ceryx:routes:{hostname}" + redis_client.set(route_redis_key, api_base_url) + + settings_redis_key = f"ceryx:settings:{hostname}" + redis_client.hset(settings_redis_key, "certificate_path", certificate_path) + redis_client.hset(settings_redis_key, "key_path", key_path) + + testing_client.get(f"https://{hostname}/", verify=certificate_path) \ No newline at end of file diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 1c8216f..784de65 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -1,4 +1,4 @@ -version: '3.5' +version: '3.7' services: ceryx: diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 60762b8..8096fb4 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -1,6 +1,10 @@ -version: '3.5' +version: '3.7' services: + ceryx: + volumes: + - test_certificates:/usr/local/share/certificates + test: build: context: ./ceryx @@ -9,6 +13,10 @@ services: CERYX_API_URL: "http://api:${CERYX_API_PORT:-5555}" volumes: - ./ceryx:/usr/src/app + - test_certificates:/usr/local/share/certificates depends_on: - ceryx - api + +volumes: + test_certificates: diff --git a/docker-compose.yml b/docker-compose.yml index 8767606..c9a3785 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.5' +version: '3.7' services: ceryx: From 72135af352004869524044d9307665939b38da3c Mon Sep 17 00:00:00 2001 From: Paris Kasidiaris Date: Wed, 24 Apr 2019 18:58:29 +0300 Subject: [PATCH 07/17] Use testing client in all tests --- ceryx/tests/client.py | 3 ++- ceryx/tests/test_certificates.py | 6 +++--- ceryx/tests/test_routes.py | 35 ++++++++++++++------------------ 3 files changed, 20 insertions(+), 24 deletions(-) diff --git a/ceryx/tests/client.py b/ceryx/tests/client.py index 9234146..637fee1 100644 --- a/ceryx/tests/client.py +++ b/ceryx/tests/client.py @@ -56,6 +56,7 @@ class CeryxTestsHTTPConnectionPool(HTTPConnectionPool): ConnectionCls = CeryxTestsHTTPConnection def __init__(self, host, *args, **kwargs): + print(f"IMPOSTOR HOST: {host}") self._impostor_host = host super().__init__(host, *args, **kwargs) @@ -96,7 +97,7 @@ def init_poolmanager( ) -class Client(Session): +class CeryxTestClient(Session): def __init__(self): super().__init__() self.mount("http://", CeryxTestsHTTPAdapter()) diff --git a/ceryx/tests/test_certificates.py b/ceryx/tests/test_certificates.py index 53fb132..497ce3d 100644 --- a/ceryx/tests/test_certificates.py +++ b/ceryx/tests/test_certificates.py @@ -6,7 +6,7 @@ import redis import requests -import client +from client import CeryxTestClient ALL_CAN_READ = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH @@ -14,7 +14,7 @@ CERYX_HOST = "ceryx" redis_client = redis.Redis(host='redis') -testing_client = client.Client() +test_client = CeryxTestClient() def test_custom_certificate(): @@ -52,4 +52,4 @@ def test_custom_certificate(): redis_client.hset(settings_redis_key, "certificate_path", certificate_path) redis_client.hset(settings_redis_key, "key_path", key_path) - testing_client.get(f"https://{hostname}/", verify=certificate_path) \ No newline at end of file + test_client.get(f"https://{hostname}/", verify=certificate_path) \ No newline at end of file diff --git a/ceryx/tests/test_routes.py b/ceryx/tests/test_routes.py index e2c6adf..d3bb244 100644 --- a/ceryx/tests/test_routes.py +++ b/ceryx/tests/test_routes.py @@ -2,20 +2,23 @@ import requests +from client import CeryxTestClient + + CERYX_API_URL = os.getenv("CERYX_API_URL", "http://api:5555") CERYX_API_ROUTES_ROOT = os.path.join(CERYX_API_URL, "api/routes") CERYX_HOST = "http://ceryx" +test_client = CeryxTestClient() + def test_no_route(): """ Ceryx should send a `503` response when receiving a request with a `Host` header that has not been registered for routing. """ - response = requests.get( - CERYX_HOST, headers={"Host": "i-do-not-exist.ceryx.test"} - ) + response = test_client.get("http://i-do-not-exist.ceryx.test/") assert response.status_code == 503 @@ -34,12 +37,8 @@ def test_proxy(): json={"source": ceryx_route_source, "target": ceryx_route_target}, ) - upstream_response = requests.get( - ceryx_route_target, headers={"Host": api_upstream_host} - ) - ceryx_response = requests.get( - f"{CERYX_HOST}", headers={"Host": ceryx_route_source} - ) + upstream_response = test_client.get(ceryx_route_target) + ceryx_response = test_client.get(f"http://{ceryx_route_source}/") assert upstream_response.status_code == ceryx_response.status_code assert upstream_response.content == ceryx_response.content @@ -64,12 +63,10 @@ def test_redirect(): }, ) - original_url = f"{CERYX_HOST}/some/path/?some=args&more=args" + url = f"http://{ceryx_route_source}/some/path/?some=args&more=args" target_url = f"{ceryx_route_target}/some/path/?some=args&more=args" - ceryx_response = requests.get( - original_url, headers={"Host": ceryx_route_source}, allow_redirects=False, - ) + ceryx_response = test_client.get(url, allow_redirects=False) assert ceryx_response.status_code == 301 assert ceryx_response.headers["Location"] == target_url @@ -94,12 +91,10 @@ def test_enforce_https(): }, ) - - original_url = f"{CERYX_HOST}/some/path/?some=args&more=args" - secure_url = f"https://{ceryx_route_source}/some/path/?some=args&more=args" - ceryx_response = requests.get( - original_url, headers={"Host": ceryx_route_source}, allow_redirects=False, - ) + base_url = f"{ceryx_route_source}/some/path/?some=args&more=args" + http_url = f"http://{base_url}" + https_url = f"https://{base_url}" + ceryx_response = test_client.get(http_url, allow_redirects=False) assert ceryx_response.status_code == 301 - assert ceryx_response.headers["Location"] == secure_url + assert ceryx_response.headers["Location"] == https_url From 42b1d331fa5eb394e31af97cb96d9d18d33f914f Mon Sep 17 00:00:00 2001 From: Paris Kasidiaris Date: Wed, 24 Apr 2019 18:58:50 +0300 Subject: [PATCH 08/17] Format --- ceryx/tests/test_certificates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ceryx/tests/test_certificates.py b/ceryx/tests/test_certificates.py index 497ce3d..e67cc02 100644 --- a/ceryx/tests/test_certificates.py +++ b/ceryx/tests/test_certificates.py @@ -8,6 +8,7 @@ from client import CeryxTestClient + ALL_CAN_READ = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH CERTIFICATE_ROOT = "/usr/local/share/certificates" From 511796ee4630e2e2f4704409d0bbbe0f4de9bcb4 Mon Sep 17 00:00:00 2001 From: Paris Kasidiaris Date: Wed, 24 Apr 2019 19:06:52 +0300 Subject: [PATCH 09/17] Organize testing client --- ceryx/tests/client.py | 104 --------------------------- ceryx/tests/client/__init__.py | 1 + ceryx/tests/client/adapters.py | 18 +++++ ceryx/tests/client/client.py | 10 +++ ceryx/tests/client/connection.py | 48 +++++++++++++ ceryx/tests/client/connectionpool.py | 23 ++++++ ceryx/tests/client/poolmanager.py | 15 ++++ 7 files changed, 115 insertions(+), 104 deletions(-) delete mode 100644 ceryx/tests/client.py create mode 100644 ceryx/tests/client/__init__.py create mode 100644 ceryx/tests/client/adapters.py create mode 100644 ceryx/tests/client/client.py create mode 100644 ceryx/tests/client/connection.py create mode 100644 ceryx/tests/client/connectionpool.py create mode 100644 ceryx/tests/client/poolmanager.py diff --git a/ceryx/tests/client.py b/ceryx/tests/client.py deleted file mode 100644 index 637fee1..0000000 --- a/ceryx/tests/client.py +++ /dev/null @@ -1,104 +0,0 @@ -import os -import socket - -from requests import Session -from requests.adapters import DEFAULT_POOLBLOCK, HTTPAdapter -from urllib3.connection import HTTPConnection, HTTPSConnection -from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool -from urllib3.poolmanager import PoolManager - - -DEFAULT_CERYX_HOST = "ceryx" # Set by Docker Compose in tests -CERYX_HOST = os.getenv("CERYX_HOST", DEFAULT_CERYX_HOST) - - -class CeryxTestsHTTPConnection(HTTPConnection): - """ - Custom-built HTTPConnection for Ceryx tests. - """ - - @property - def host(self): - return self._dns_host.rstrip('.') - - @host.setter - def host(self, value): - """ - Do exactly what the parent class does. - """ - self._dns_host = CERYX_HOST if value.endswith(".ceryx.test") else value - - -class CeryxTestsHTTPSConnection(CeryxTestsHTTPConnection, HTTPSConnection): - def __init__( - self, host, port=None, key_file=None, cert_file=None, - key_password=None, strict=None, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT, ssl_context=None, - server_hostname=None, **kw, - ): - - CeryxTestsHTTPConnection.__init__( - self, host, port, strict=strict, timeout=timeout, **kw, - ) - - self.key_file = key_file - self.cert_file = cert_file - self.key_password = key_password - self.ssl_context = ssl_context - self.server_hostname = server_hostname - - # Required property for Google AppEngine 1.9.0 which otherwise causes - # HTTPS requests to go out as HTTP. (See Issue #356) - self._protocol = 'https' - - -class CeryxTestsHTTPConnectionPool(HTTPConnectionPool): - ConnectionCls = CeryxTestsHTTPConnection - - def __init__(self, host, *args, **kwargs): - print(f"IMPOSTOR HOST: {host}") - self._impostor_host = host - super().__init__(host, *args, **kwargs) - - def urlopen(self, *args, **kwargs): - kwargs["headers"]["Host"] = self._impostor_host - return super().urlopen(*args, **kwargs) - - -class CeryxTestsHTTPSConnectionPool(HTTPSConnectionPool): - ConnectionCls = CeryxTestsHTTPSConnection - - def __init__(self, host, *args, **kwargs): - super().__init__(host, *args, **kwargs) - self.conn_kw["server_hostname"] = host - - -class CeryxTestsPoolManager(PoolManager): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.pool_classes_by_scheme = { - "http": CeryxTestsHTTPConnectionPool, - "https": CeryxTestsHTTPSConnectionPool, - } - - -class CeryxTestsHTTPAdapter(HTTPAdapter): - def init_poolmanager( - self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs, - ): - # Comment from original Requests HTTPAdapter: Save these values for pickling - self._pool_connections = connections - self._pool_maxsize = maxsize - self._pool_block = block - - self.poolmanager = CeryxTestsPoolManager( - num_pools=connections, maxsize=maxsize, block=block, strict=True, - **pool_kwargs, - ) - - -class CeryxTestClient(Session): - def __init__(self): - super().__init__() - self.mount("http://", CeryxTestsHTTPAdapter()) - self.mount("https://", CeryxTestsHTTPAdapter()) diff --git a/ceryx/tests/client/__init__.py b/ceryx/tests/client/__init__.py new file mode 100644 index 0000000..a3f9dc1 --- /dev/null +++ b/ceryx/tests/client/__init__.py @@ -0,0 +1 @@ +from .client import CeryxTestClient \ No newline at end of file diff --git a/ceryx/tests/client/adapters.py b/ceryx/tests/client/adapters.py new file mode 100644 index 0000000..2894f36 --- /dev/null +++ b/ceryx/tests/client/adapters.py @@ -0,0 +1,18 @@ +from requests.adapters import DEFAULT_POOLBLOCK, HTTPAdapter + +from .poolmanager import CeryxTestsPoolManager + + +class CeryxTestsHTTPAdapter(HTTPAdapter): + def init_poolmanager( + self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs, + ): + # Comment from original Requests HTTPAdapter: Save these values for pickling + self._pool_connections = connections + self._pool_maxsize = maxsize + self._pool_block = block + + self.poolmanager = CeryxTestsPoolManager( + num_pools=connections, maxsize=maxsize, block=block, strict=True, + **pool_kwargs, + ) \ No newline at end of file diff --git a/ceryx/tests/client/client.py b/ceryx/tests/client/client.py new file mode 100644 index 0000000..31a96e9 --- /dev/null +++ b/ceryx/tests/client/client.py @@ -0,0 +1,10 @@ +from requests import Session + +from .adapters import CeryxTestsHTTPAdapter + + +class CeryxTestClient(Session): + def __init__(self): + super().__init__() + self.mount("http://", CeryxTestsHTTPAdapter()) + self.mount("https://", CeryxTestsHTTPAdapter()) diff --git a/ceryx/tests/client/connection.py b/ceryx/tests/client/connection.py new file mode 100644 index 0000000..bf435d8 --- /dev/null +++ b/ceryx/tests/client/connection.py @@ -0,0 +1,48 @@ +from urllib3.connection import HTTPConnection, HTTPSConnection +import os +import socket + + +DEFAULT_CERYX_HOST = "ceryx" # Set by Docker Compose in tests +CERYX_HOST = os.getenv("CERYX_HOST", DEFAULT_CERYX_HOST) + + +class CeryxTestsHTTPConnection(HTTPConnection): + """ + Custom-built HTTPConnection for Ceryx tests. + """ + + @property + def host(self): + return self._dns_host.rstrip('.') + + @host.setter + def host(self, value): + """ + Do exactly what the parent class does. + """ + self._dns_host = CERYX_HOST if value.endswith(".ceryx.test") else value + + +class CeryxTestsHTTPSConnection(CeryxTestsHTTPConnection, HTTPSConnection): + def __init__( + self, host, port=None, key_file=None, cert_file=None, + key_password=None, strict=None, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT, ssl_context=None, + server_hostname=None, **kw, + ): + + CeryxTestsHTTPConnection.__init__( + self, host, port, strict=strict, timeout=timeout, **kw, + ) + + self.key_file = key_file + self.cert_file = cert_file + self.key_password = key_password + self.ssl_context = ssl_context + self.server_hostname = server_hostname + + # Required property for Google AppEngine 1.9.0 which otherwise causes + # HTTPS requests to go out as HTTP. (See Issue #356) + self._protocol = 'https' + \ No newline at end of file diff --git a/ceryx/tests/client/connectionpool.py b/ceryx/tests/client/connectionpool.py new file mode 100644 index 0000000..02f32dc --- /dev/null +++ b/ceryx/tests/client/connectionpool.py @@ -0,0 +1,23 @@ +from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool + +from .connection import CeryxTestsHTTPConnection, CeryxTestsHTTPSConnection + + +class CeryxTestsHTTPConnectionPool(HTTPConnectionPool): + ConnectionCls = CeryxTestsHTTPConnection + + def __init__(self, host, *args, **kwargs): + self._impostor_host = host + super().__init__(host, *args, **kwargs) + + def urlopen(self, *args, **kwargs): + kwargs["headers"]["Host"] = self._impostor_host + return super().urlopen(*args, **kwargs) + + +class CeryxTestsHTTPSConnectionPool(HTTPSConnectionPool): + ConnectionCls = CeryxTestsHTTPSConnection + + def __init__(self, host, *args, **kwargs): + super().__init__(host, *args, **kwargs) + self.conn_kw["server_hostname"] = host diff --git a/ceryx/tests/client/poolmanager.py b/ceryx/tests/client/poolmanager.py new file mode 100644 index 0000000..d4503ba --- /dev/null +++ b/ceryx/tests/client/poolmanager.py @@ -0,0 +1,15 @@ +from urllib3.poolmanager import PoolManager + +from .connectionpool import ( + CeryxTestsHTTPConnectionPool, + CeryxTestsHTTPSConnectionPool, +) + + +class CeryxTestsPoolManager(PoolManager): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.pool_classes_by_scheme = { + "http": CeryxTestsHTTPConnectionPool, + "https": CeryxTestsHTTPSConnectionPool, + } \ No newline at end of file From 7c1946a1a85c77cade3b07264c41315f163da071 Mon Sep 17 00:00:00 2001 From: Paris Kasidiaris Date: Wed, 24 Apr 2019 19:17:02 +0300 Subject: [PATCH 10/17] Documentation. --- ceryx/tests/client/client.py | 9 +++++++++ ceryx/tests/client/connection.py | 16 ++++++++++++++-- ceryx/tests/client/connectionpool.py | 11 +++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/ceryx/tests/client/client.py b/ceryx/tests/client/client.py index 31a96e9..9fe5e47 100644 --- a/ceryx/tests/client/client.py +++ b/ceryx/tests/client/client.py @@ -4,6 +4,15 @@ class CeryxTestClient(Session): + """ + The Ceryx testing client lets us test Ceryx hosts without any + configuration. Essentially lets us make requests to + hostnames ending in `.ceryx.test`, without any name resolution + needed. The testing client will make these requests to the configured + Ceryx host automatically, but will set both the `Host` HTTP header + and `SNI` SSL attribute to the initial host. + """ + def __init__(self): super().__init__() self.mount("http://", CeryxTestsHTTPAdapter()) diff --git a/ceryx/tests/client/connection.py b/ceryx/tests/client/connection.py index bf435d8..8dae9d7 100644 --- a/ceryx/tests/client/connection.py +++ b/ceryx/tests/client/connection.py @@ -9,17 +9,24 @@ class CeryxTestsHTTPConnection(HTTPConnection): """ - Custom-built HTTPConnection for Ceryx tests. + Custom-built HTTPConnection for Ceryx tests. Force sets the request's + host to the configured Ceryx host, if the request's original host + ends with `.ceryx.test`. """ @property def host(self): + """ + Do what the original property did. We just want to touch the setter. + """ return self._dns_host.rstrip('.') @host.setter def host(self, value): """ - Do exactly what the parent class does. + If the request header ends with `.ceryx.test` then force set the actual + host to the configured Ceryx host, so as to send corresponding + requests to Ceryx. """ self._dns_host = CERYX_HOST if value.endswith(".ceryx.test") else value @@ -32,6 +39,7 @@ def __init__( server_hostname=None, **kw, ): + # Initialise the HTTPConnection subclass created above. CeryxTestsHTTPConnection.__init__( self, host, port, strict=strict, timeout=timeout, **kw, ) @@ -42,6 +50,10 @@ def __init__( self.ssl_context = ssl_context self.server_hostname = server_hostname + # ------------------------------ + # Original comment from upstream + # ------------------------------ + # # Required property for Google AppEngine 1.9.0 which otherwise causes # HTTPS requests to go out as HTTP. (See Issue #356) self._protocol = 'https' diff --git a/ceryx/tests/client/connectionpool.py b/ceryx/tests/client/connectionpool.py index 02f32dc..6c0e4ec 100644 --- a/ceryx/tests/client/connectionpool.py +++ b/ceryx/tests/client/connectionpool.py @@ -7,10 +7,18 @@ class CeryxTestsHTTPConnectionPool(HTTPConnectionPool): ConnectionCls = CeryxTestsHTTPConnection def __init__(self, host, *args, **kwargs): + """ + Store the original HTTP request host, so we can pass it over via the + `Host` header. + """ self._impostor_host = host super().__init__(host, *args, **kwargs) def urlopen(self, *args, **kwargs): + """ + This custom `urlopen` implementation enforces setting the `Host` header + of the request to `self._impostor_host`. + """ kwargs["headers"]["Host"] = self._impostor_host return super().urlopen(*args, **kwargs) @@ -19,5 +27,8 @@ class CeryxTestsHTTPSConnectionPool(HTTPSConnectionPool): ConnectionCls = CeryxTestsHTTPSConnection def __init__(self, host, *args, **kwargs): + """ + Force set SNI to the requested Host. + """ super().__init__(host, *args, **kwargs) self.conn_kw["server_hostname"] = host From 1775ebaba1164bac34b1efa9ec2d35ed40f4cef9 Mon Sep 17 00:00:00 2001 From: Paris Kasidiaris Date: Wed, 24 Apr 2019 19:22:32 +0300 Subject: [PATCH 11/17] Extract certificate creation to utils. --- ceryx/tests/test_certificates.py | 32 ++++++-------------------------- ceryx/tests/utils.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 26 deletions(-) create mode 100644 ceryx/tests/utils.py diff --git a/ceryx/tests/test_certificates.py b/ceryx/tests/test_certificates.py index e67cc02..bbd62dd 100644 --- a/ceryx/tests/test_certificates.py +++ b/ceryx/tests/test_certificates.py @@ -7,11 +7,9 @@ import requests from client import CeryxTestClient +from utils import create_certificates_for_host -ALL_CAN_READ = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH - -CERTIFICATE_ROOT = "/usr/local/share/certificates" CERYX_HOST = "ceryx" redis_client = redis.Redis(host='redis') @@ -23,34 +21,16 @@ def test_custom_certificate(): Ensure that Ceryx uses the given certificate for each route, if configured so. """ - certificate_id = uuid.uuid4() - certificate_path = f"{CERTIFICATE_ROOT}/{certificate_id}.crt" - key_path = f"{CERTIFICATE_ROOT}/{certificate_id}.key" - hostname = "custom-certificate.ceryx.test" - - command = [ - "openssl", - "req", "-x509", - "-newkey", "rsa:4096", - "-keyout", key_path, - "-out", certificate_path, - "-days", "1", - "-subj", f"/C=GR/ST=Attica/L=Athens/O=SourceLair/OU=Org/CN={hostname}", - "-nodes", - ] - subprocess.run( - command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, - ) - os.chmod(certificate_path, ALL_CAN_READ) - os.chmod(key_path, ALL_CAN_READ) + host = f"{uuid.uuid4()}.ceryx.test" + certificate_path , key_path = create_certificates_for_host(host) api_base_url = "http://api:5555/" - route_redis_key = f"ceryx:routes:{hostname}" + route_redis_key = f"ceryx:routes:{host}" redis_client.set(route_redis_key, api_base_url) - settings_redis_key = f"ceryx:settings:{hostname}" + settings_redis_key = f"ceryx:settings:{host}" redis_client.hset(settings_redis_key, "certificate_path", certificate_path) redis_client.hset(settings_redis_key, "key_path", key_path) - test_client.get(f"https://{hostname}/", verify=certificate_path) \ No newline at end of file + test_client.get(f"https://{host}/", verify=certificate_path) diff --git a/ceryx/tests/utils.py b/ceryx/tests/utils.py new file mode 100644 index 0000000..0eee4ee --- /dev/null +++ b/ceryx/tests/utils.py @@ -0,0 +1,31 @@ +import os +import stat +import subprocess + + +CERTIFICATE_ROOT = "/usr/local/share/certificates" +EVERYBODY_CAN_READ = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH + + +def create_certificates_for_host(host): + base_path = f"{CERTIFICATE_ROOT}/{host}" + certificate_path = f"{base_path}.crt" + key_path = f"{base_path}.key" + + command = [ + "openssl", + "req", "-x509", + "-newkey", "rsa:4096", + "-keyout", key_path, + "-out", certificate_path, + "-days", "1", + "-subj", f"/C=GR/ST=Attica/L=Athens/O=SourceLair/OU=Org/CN={host}", + "-nodes", + ] + subprocess.run( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, + ) + os.chmod(certificate_path, EVERYBODY_CAN_READ) + os.chmod(key_path, EVERYBODY_CAN_READ) + + return certificate_path, key_path From cbe536fbe8dd2b416f81ee3ebe875cbeffb882f1 Mon Sep 17 00:00:00 2001 From: Paris Kasidiaris Date: Wed, 24 Apr 2019 20:03:37 +0300 Subject: [PATCH 12/17] Add class-based tests --- ceryx/tests/base.py | 14 +++ ceryx/tests/test_certificates.py | 44 +++----- ceryx/tests/test_routes.py | 177 +++++++++++++++---------------- 3 files changed, 117 insertions(+), 118 deletions(-) create mode 100644 ceryx/tests/base.py diff --git a/ceryx/tests/base.py b/ceryx/tests/base.py new file mode 100644 index 0000000..cb2f8af --- /dev/null +++ b/ceryx/tests/base.py @@ -0,0 +1,14 @@ +import os +import uuid + +import redis + +from client import CeryxTestClient + + +class BaseTest: + def setup_method(self): + self.uuid = uuid.uuid4() + self.host = f"{self.uuid}.ceryx.test" + self.client = CeryxTestClient() + self.redis = redis.Redis(host='redis') \ No newline at end of file diff --git a/ceryx/tests/test_certificates.py b/ceryx/tests/test_certificates.py index bbd62dd..8e7d836 100644 --- a/ceryx/tests/test_certificates.py +++ b/ceryx/tests/test_certificates.py @@ -1,36 +1,22 @@ -import os -import stat -import subprocess -import uuid - -import redis -import requests - -from client import CeryxTestClient +from base import BaseTest from utils import create_certificates_for_host -CERYX_HOST = "ceryx" - -redis_client = redis.Redis(host='redis') -test_client = CeryxTestClient() - - -def test_custom_certificate(): - """ - Ensure that Ceryx uses the given certificate for each route, if configured - so. - """ - host = f"{uuid.uuid4()}.ceryx.test" - certificate_path , key_path = create_certificates_for_host(host) +class TestCertificates(BaseTest): + def test_custom_certificate(self): + """ + Ensure that Ceryx uses the given certificate for each route, if configured + so. + """ + certificate_path , key_path = create_certificates_for_host(self.host) - api_base_url = "http://api:5555/" + api_base_url = "http://api:5555/" - route_redis_key = f"ceryx:routes:{host}" - redis_client.set(route_redis_key, api_base_url) + route_redis_key = f"ceryx:routes:{self.host}" + self.redis.set(route_redis_key, api_base_url) - settings_redis_key = f"ceryx:settings:{host}" - redis_client.hset(settings_redis_key, "certificate_path", certificate_path) - redis_client.hset(settings_redis_key, "key_path", key_path) + settings_redis_key = f"ceryx:settings:{self.host}" + self.redis.hset(settings_redis_key, "certificate_path", certificate_path) + self.redis.hset(settings_redis_key, "key_path", key_path) - test_client.get(f"https://{host}/", verify=certificate_path) + self.client.get(f"https://{self.host}/", verify=certificate_path) diff --git a/ceryx/tests/test_routes.py b/ceryx/tests/test_routes.py index d3bb244..19982c4 100644 --- a/ceryx/tests/test_routes.py +++ b/ceryx/tests/test_routes.py @@ -2,7 +2,7 @@ import requests -from client import CeryxTestClient +from base import BaseTest CERYX_API_URL = os.getenv("CERYX_API_URL", "http://api:5555") @@ -10,91 +10,90 @@ CERYX_HOST = "http://ceryx" -test_client = CeryxTestClient() - - -def test_no_route(): - """ - Ceryx should send a `503` response when receiving a request with a `Host` - header that has not been registered for routing. - """ - response = test_client.get("http://i-do-not-exist.ceryx.test/") - assert response.status_code == 503 - - -def test_proxy(): - """ - Ceryx should successfully proxy the upstream request to the client, for a - registered route. - """ - api_upstream_host = "api" - ceryx_route_source = "api.ceryx.test" - ceryx_route_target = f"http://{api_upstream_host}:5555/api/routes" - - # Register the local Ceryx API as a route - register_api_response = requests.post( - CERYX_API_ROUTES_ROOT, - json={"source": ceryx_route_source, "target": ceryx_route_target}, - ) - - upstream_response = test_client.get(ceryx_route_target) - ceryx_response = test_client.get(f"http://{ceryx_route_source}/") - - assert upstream_response.status_code == ceryx_response.status_code - assert upstream_response.content == ceryx_response.content - - -def test_redirect(): - """ - Ceryx should respond with 301 status and the appropriate `Location` header - for redirected routes. - """ - api_upstream_host = "api" - ceryx_route_target = "http://api:5555/api/routes" - ceryx_route_source = "redirected-api.ceryx.test" - - # Register the local Ceryx API as a route - register_api_response = requests.post( - CERYX_API_ROUTES_ROOT, - json={ - "source": ceryx_route_source, - "target": ceryx_route_target, - "settings": {"mode": "redirect"}, - }, - ) - - url = f"http://{ceryx_route_source}/some/path/?some=args&more=args" - target_url = f"{ceryx_route_target}/some/path/?some=args&more=args" - - ceryx_response = test_client.get(url, allow_redirects=False) - - assert ceryx_response.status_code == 301 - assert ceryx_response.headers["Location"] == target_url - - -def test_enforce_https(): - """ - Ceryx should respond with 301 status and the appropriate `Location` header - for routes with HTTPS enforced. - """ - api_upstream_host = "api" - api_upstream_target = "http://api:5555/" - ceryx_route_source = "secure-api.ceryx.test" - - # Register the local Ceryx API as a route - register_api_response = requests.post( - CERYX_API_ROUTES_ROOT, - json={ - "source": ceryx_route_source, - "target": api_upstream_target, - "settings": {"enforce_https": True}, - }, - ) - - base_url = f"{ceryx_route_source}/some/path/?some=args&more=args" - http_url = f"http://{base_url}" - https_url = f"https://{base_url}" - ceryx_response = test_client.get(http_url, allow_redirects=False) - - assert ceryx_response.status_code == 301 - assert ceryx_response.headers["Location"] == https_url + +class TestRoutes(BaseTest): + def test_no_route(self): + """ + Ceryx should send a `503` response when receiving a request with a `Host` + header that has not been registered for routing. + """ + response = self.client.get("http://i-do-not-exist.ceryx.test/") + assert response.status_code == 503 + + + def test_proxy(self): + """ + Ceryx should successfully proxy the upstream request to the client, for a + registered route. + """ + api_upstream_host = "api" + ceryx_route_source = "api.ceryx.test" + ceryx_route_target = f"http://{api_upstream_host}:5555/api/routes" + + # Register the local Ceryx API as a route + register_api_response = requests.post( + CERYX_API_ROUTES_ROOT, + json={"source": ceryx_route_source, "target": ceryx_route_target}, + ) + + upstream_response = self.client.get(ceryx_route_target) + ceryx_response = self.client.get(f"http://{ceryx_route_source}/") + + assert upstream_response.status_code == ceryx_response.status_code + assert upstream_response.content == ceryx_response.content + + + def test_redirect(self): + """ + Ceryx should respond with 301 status and the appropriate `Location` header + for redirected routes. + """ + api_upstream_host = "api" + ceryx_route_target = "http://api:5555/api/routes" + ceryx_route_source = "redirected-api.ceryx.test" + + # Register the local Ceryx API as a route + register_api_response = requests.post( + CERYX_API_ROUTES_ROOT, + json={ + "source": ceryx_route_source, + "target": ceryx_route_target, + "settings": {"mode": "redirect"}, + }, + ) + + url = f"http://{ceryx_route_source}/some/path/?some=args&more=args" + target_url = f"{ceryx_route_target}/some/path/?some=args&more=args" + + ceryx_response = self.client.get(url, allow_redirects=False) + + assert ceryx_response.status_code == 301 + assert ceryx_response.headers["Location"] == target_url + + + def test_enforce_https(self): + """ + Ceryx should respond with 301 status and the appropriate `Location` header + for routes with HTTPS enforced. + """ + api_upstream_host = "api" + api_upstream_target = "http://api:5555/" + ceryx_route_source = "secure-api.ceryx.test" + + # Register the local Ceryx API as a route + register_api_response = requests.post( + CERYX_API_ROUTES_ROOT, + json={ + "source": ceryx_route_source, + "target": api_upstream_target, + "settings": {"enforce_https": True}, + }, + ) + + base_url = f"{ceryx_route_source}/some/path/?some=args&more=args" + http_url = f"http://{base_url}" + https_url = f"https://{base_url}" + ceryx_response = self.client.get(http_url, allow_redirects=False) + + assert ceryx_response.status_code == 301 + assert ceryx_response.headers["Location"] == https_url From caa9c526c8f6069db56d590a65b23d597b1f5031 Mon Sep 17 00:00:00 2001 From: Paris Kasidiaris Date: Thu, 25 Apr 2019 11:51:53 +0300 Subject: [PATCH 13/17] Start migrating to Responder and better Redis/API serialisation --- api/Dockerfile | 3 +- api/Pipfile | 2 + api/Pipfile.lock | 270 +++++++++++++++++++++++++++++++++--- api/api.py | 13 ++ api/app.py | 1 + api/ceryx/db.py | 113 +++++++-------- api/ceryx/schemas.py | 73 ++++++++++ api/ceryx/types.py | 9 +- docker-compose.override.yml | 1 + 9 files changed, 401 insertions(+), 84 deletions(-) create mode 100644 api/api.py create mode 100644 api/ceryx/schemas.py diff --git a/api/Dockerfile b/api/Dockerfile index 74db173..e74049b 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -11,4 +11,5 @@ RUN pipenv install --system --dev --deploy COPY . /opt/ceryx WORKDIR /opt/ceryx -CMD python app.py +ENV PORT 5555 +CMD python api.py diff --git a/api/Pipfile b/api/Pipfile index 18413a0..8b0a525 100644 --- a/api/Pipfile +++ b/api/Pipfile @@ -7,6 +7,8 @@ name = "pypi" apistar = "==0.4.3" redis = "*" requests = ">=2.21.0" +typesystem = "*" +responder = "*" [dev-packages] nose = "*" diff --git a/api/Pipfile.lock b/api/Pipfile.lock index dfb63cb..8f833bd 100644 --- a/api/Pipfile.lock +++ b/api/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "5c056fe85a233763e10d9f421623a691326fe7fab0d634a792f47bb69b6f5505" + "sha256": "c96798be36a81423cba68109512b6b8cb61e43e6ba62c6626eb8cf87ba8dc9b5" }, "pipfile-spec": 6, "requires": { @@ -16,6 +16,27 @@ ] }, "default": { + "aiofiles": { + "hashes": [ + "sha256:021ea0ba314a86027c166ecc4b4c07f2d40fc0f4b3a950d1868a0f2571c2bbee", + "sha256:1e644c2573f953664368de28d2aa4c89dfd64550429d0c27c4680ccd3aa4985d" + ], + "version": "==0.4.0" + }, + "aniso8601": { + "hashes": [ + "sha256:7849749cf00ae0680ad2bdfe4419c7a662bef19c03691a19e008c8b9a5267802", + "sha256:94f90871fcd314a458a3d4eca1c84448efbd200e86f55fe4c733c7a40149ef50" + ], + "version": "==3.0.2" + }, + "apispec": { + "hashes": [ + "sha256:9f7323abd9f0bbb12f98a155a9ec436a048897d3550babc935664f3dc26ad507", + "sha256:a350fa2f87d1462acc4b3a52ce2ddaf04805b44911e053ee0a68eb50919ea690" + ], + "version": "==1.3.0" + }, "apistar": { "hashes": [ "sha256:e4c82c8c1467a4a76ddf431b65686aebe89f37cefeb320dc8dcebfeb5928ab20" @@ -23,12 +44,26 @@ "index": "pypi", "version": "==0.4.3" }, + "asgiref": { + "hashes": [ + "sha256:48afe222aefece5814ae90aae394964eada5a4604e67f9397f7858e8957e9fdf", + "sha256:60c783a7994246b2e710aa2f0a2f7fcfacf156cffc7b50f7074bfd97c9046db3" + ], + "version": "==3.1.2" + }, + "async-timeout": { + "hashes": [ + "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f", + "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3" + ], + "version": "==3.0.1" + }, "certifi": { "hashes": [ - "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", - "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", + "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" ], - "version": "==2018.11.29" + "version": "==2019.3.9" }, "chardet": { "hashes": [ @@ -37,6 +72,58 @@ ], "version": "==3.0.4" }, + "click": { + "hashes": [ + "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", + "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" + ], + "version": "==7.0" + }, + "docopt": { + "hashes": [ + "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" + ], + "version": "==0.6.2" + }, + "graphene": { + "hashes": [ + "sha256:b8ec446d17fa68721636eaad3d6adc1a378cb6323e219814c8f98c9928fc9642", + "sha256:faa26573b598b22ffd274e2fd7a4c52efa405dcca96e01a62239482246248aa3" + ], + "version": "==2.1.3" + }, + "graphql-core": { + "hashes": [ + "sha256:889e869be5574d02af77baf1f30b5db9ca2959f1c9f5be7b2863ead5a3ec6181", + "sha256:9462e22e32c7f03b667373ec0a84d95fba10e8ce2ead08f29fbddc63b671b0c1" + ], + "version": "==2.1" + }, + "graphql-relay": { + "hashes": [ + "sha256:2716b7245d97091af21abf096fabafac576905096d21ba7118fba722596f65db" + ], + "version": "==0.4.5" + }, + "graphql-server-core": { + "hashes": [ + "sha256:e5f82add4b3d5580aa1f1e7d9f00e944ad3abe1b65eb337e611d6a77cc20f231" + ], + "version": "==1.1.1" + }, + "h11": { + "hashes": [ + "sha256:acca6a44cb52a32ab442b1779adf0875c443c689e9e028f8d831a3769f9c5208", + "sha256:f2b1ca39bfed357d1f19ac732913d5f9faa54a5062eca7d2ec3a916cfb7ae4c7" + ], + "version": "==0.8.1" + }, + "httptools": { + "hashes": [ + "sha256:e00cbd7ba01ff748e494248183abc6e153f49181169d8a3d41bb49132ca01dfc" + ], + "version": "==0.0.13" + }, "idna": { "hashes": [ "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", @@ -44,12 +131,19 @@ ], "version": "==2.8" }, + "itsdangerous": { + "hashes": [ + "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", + "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" + ], + "version": "==1.1.0" + }, "jinja2": { "hashes": [ - "sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd", - "sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4" + "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", + "sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b" ], - "version": "==2.10" + "version": "==2.10.1" }, "markupsafe": { "hashes": [ @@ -84,13 +178,55 @@ ], "version": "==1.1.1" }, + "marshmallow": { + "hashes": [ + "sha256:0e497a6447ffaad55578138ca512752de7a48d12f444996ededc3d6bf8a09ca2", + "sha256:e21a4dea20deb167c723e0ffb13f4cf33bcbbeb8a334e92406a3308cedea2826" + ], + "version": "==2.19.2" + }, + "parse": { + "hashes": [ + "sha256:1b68657434d371e5156048ca4a0c5aea5afc6ca59a2fea4dd1a575354f617142" + ], + "version": "==1.12.0" + }, + "promise": { + "hashes": [ + "sha256:2ebbfc10b7abf6354403ed785fe4f04b9dfd421eb1a474ac8d187022228332af", + "sha256:348f5f6c3edd4fd47c9cd65aed03ac1b31136d375aa63871a57d3e444c85655c" + ], + "version": "==2.2.1" + }, + "python-multipart": { + "hashes": [ + "sha256:f7bb5f611fc600d15fa47b3974c8aa16e93724513b49b5f95c81e6624c83fa43" + ], + "version": "==0.0.5" + }, + "pyyaml": { + "hashes": [ + "sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c", + "sha256:436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95", + "sha256:460a5a4248763f6f37ea225d19d5c205677d8d525f6a83357ca622ed541830c2", + "sha256:5a22a9c84653debfbf198d02fe592c176ea548cccce47553f35f466e15cf2fd4", + "sha256:7a5d3f26b89d688db27822343dfa25c599627bc92093e788956372285c6298ad", + "sha256:9372b04a02080752d9e6f990179a4ab840227c6e2ce15b95e1278456664cf2ba", + "sha256:a5dcbebee834eaddf3fa7366316b880ff4062e4bcc9787b78c7fbb4a26ff2dd1", + "sha256:aee5bab92a176e7cd034e57f46e9df9a9862a71f8f37cad167c6fc74c65f5b4e", + "sha256:c51f642898c0bacd335fc119da60baae0824f2cde95b0330b56c0553439f0673", + "sha256:c68ea4d3ba1705da1e0d85da6684ac657912679a649e8868bd850d2c299cce13", + "sha256:e23d0cc5299223dcc37885dae624f382297717e459ea24053709675a976a3e19" + ], + "version": "==5.1" + }, "redis": { "hashes": [ - "sha256:724932360d48e5407e8f82e405ab3650a36ed02c7e460d1e6fddf0f038422b54", - "sha256:9b19425a38fd074eb5795ff2b0d9a55b46a44f91f5347995f27e3ad257a7d775" + "sha256:6946b5dca72e86103edc8033019cc3814c031232d339d5f4533b02ea85685175", + "sha256:8ca418d2ddca1b1a850afa1680a7d2fd1f3322739271de4b704e0d4668449273" ], "index": "pypi", - "version": "==3.2.0" + "version": "==3.2.1" }, "requests": { "hashes": [ @@ -100,19 +236,115 @@ "index": "pypi", "version": "==2.21.0" }, + "requests-toolbelt": { + "hashes": [ + "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f", + "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0" + ], + "version": "==0.9.1" + }, + "responder": { + "hashes": [ + "sha256:8418d015874ad82ddb2da31c4fe82ca42a7d62462325097d79ceb907c0622e02", + "sha256:a18454d517551d2788acbac2557948ea6729d0c837a676e3ff7a57863190743d" + ], + "index": "pypi", + "version": "==1.3.0" + }, + "rfc3986": { + "hashes": [ + "sha256:2cb285760d8ed6683f9a242686961918d555f6783027d596cb82df51bfa0f9ca", + "sha256:a69146f5014a7da1fed9d375c99f5fe2782a27c0e75c778a4083fe954abbde42" + ], + "version": "==1.3.1" + }, + "rx": { + "hashes": [ + "sha256:13a1d8d9e252625c173dc795471e614eadfe1cf40ffc684e08b8fff0d9748c23", + "sha256:7357592bc7e881a95e0c2013b73326f704953301ab551fbc8133a6fadab84105" + ], + "version": "==1.6.1" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "starlette": { + "hashes": [ + "sha256:8bc2e41f7638290379ae91450413796f92d6c97b88a6b754f3c1a7f8bc7a07d6" + ], + "version": "==0.10.7" + }, + "typesystem": { + "hashes": [ + "sha256:aa01ac52370a7e5996960c8a899da0f939753bc49d405e92dea5cb1f6bc3700a" + ], + "index": "pypi", + "version": "==0.2.2" + }, "urllib3": { "hashes": [ - "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", - "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", + "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3" + ], + "version": "==1.24.2" + }, + "uvicorn": { + "hashes": [ + "sha256:181d47abddedd0f6e23eaeed97976bdce9ea1dbff0ec12385309cf4835783f6a" ], - "version": "==1.24.1" + "version": "==0.7.0" + }, + "uvloop": { + "hashes": [ + "sha256:0fcd894f6fc3226a962ee7ad895c4f52e3f5c3c55098e21efb17c071849a0573", + "sha256:2f31de1742c059c96cb76b91c5275b22b22b965c886ee1fced093fa27dde9e64", + "sha256:459e4649fcd5ff719523de33964aa284898e55df62761e7773d088823ccbd3e0", + "sha256:67867aafd6e0bc2c30a079603a85d83b94f23c5593b3cc08ec7e58ac18bf48e5", + "sha256:8c200457e6847f28d8bb91c5e5039d301716f5f2fce25646f5fb3fd65eda4a26", + "sha256:958906b9ca39eb158414fbb7d6b8ef1b7aee4db5c8e8e5d00fcbb69a1ce9dca7", + "sha256:ac1dca3d8f3ef52806059e81042ee397ac939e5a86c8a3cea55d6b087db66115", + "sha256:b284c22d8938866318e3b9d178142b8be316c52d16fcfe1560685a686718a021", + "sha256:c48692bf4587ce281d641087658eca275a5ad3b63c78297bbded96570ae9ce8f", + "sha256:fefc3b2b947c99737c348887db2c32e539160dcbeb7af9aa6b53db7a283538fe" + ], + "version": "==0.12.2" + }, + "websockets": { + "hashes": [ + "sha256:04b42a1b57096ffa5627d6a78ea1ff7fad3bc2c0331ffc17bc32a4024da7fea0", + "sha256:08e3c3e0535befa4f0c4443824496c03ecc25062debbcf895874f8a0b4c97c9f", + "sha256:10d89d4326045bf5e15e83e9867c85d686b612822e4d8f149cf4840aab5f46e0", + "sha256:232fac8a1978fc1dead4b1c2fa27c7756750fb393eb4ac52f6bc87ba7242b2fa", + "sha256:4bf4c8097440eff22bc78ec76fe2a865a6e658b6977a504679aaf08f02c121da", + "sha256:51642ea3a00772d1e48fb0c492f0d3ae3b6474f34d20eca005a83f8c9c06c561", + "sha256:55d86102282a636e195dad68aaaf85b81d0bef449d7e2ef2ff79ac450bb25d53", + "sha256:564d2675682bd497b59907d2205031acbf7d3fadf8c763b689b9ede20300b215", + "sha256:5d13bf5197a92149dc0badcc2b699267ff65a867029f465accfca8abab95f412", + "sha256:5eda665f6789edb9b57b57a159b9c55482cbe5b046d7db458948370554b16439", + "sha256:5edb2524d4032be4564c65dc4f9d01e79fe8fad5f966e5b552f4e5164fef0885", + "sha256:79691794288bc51e2a3b8de2bc0272ca8355d0b8503077ea57c0716e840ebaef", + "sha256:7fcc8681e9981b9b511cdee7c580d5b005f3bb86b65bde2188e04a29f1d63317", + "sha256:8e447e05ec88b1b408a4c9cde85aa6f4b04f06aa874b9f0b8e8319faf51b1fee", + "sha256:90ea6b3e7787620bb295a4ae050d2811c807d65b1486749414f78cfd6fb61489", + "sha256:9e13239952694b8b831088431d15f771beace10edfcf9ef230cefea14f18508f", + "sha256:d40f081187f7b54d7a99d8a5c782eaa4edc335a057aa54c85059272ed826dc09", + "sha256:e1df1a58ed2468c7b7ce9a2f9752a32ad08eac2bcd56318625c3647c2cd2da6f", + "sha256:e98d0cec437097f09c7834a11c69d79fe6241729b23f656cfc227e93294fc242", + "sha256:f8d59627702d2ff27cb495ca1abdea8bd8d581de425c56e93bff6517134e0a9b", + "sha256:fc30cdf2e949a2225b012a7911d1d031df3d23e99b7eda7dfc982dc4a860dae9" + ], + "version": "==7.0" }, "werkzeug": { "hashes": [ - "sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c", - "sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b" + "sha256:0a73e8bb2ff2feecfc5d56e6f458f5b99290ef34f565ffb2665801ff7de6af7a", + "sha256:7fad9770a8778f9576693f0cc29c7dcc36964df916b83734f4431c0e612a7fbc" ], - "version": "==0.14.1" + "version": "==0.15.2" }, "whitenoise": { "hashes": [ @@ -132,10 +364,10 @@ }, "attrs": { "hashes": [ - "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", - "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" ], - "version": "==18.2.0" + "version": "==19.1.0" }, "black": { "hashes": [ diff --git a/api/api.py b/api/api.py new file mode 100644 index 0000000..5138a85 --- /dev/null +++ b/api/api.py @@ -0,0 +1,13 @@ +import responder + +from ceryx.db import RedisClient + +api = responder.API() +client = RedisClient.from_config() + +@api.route("/api/routes/") +async def list_routes(req, resp): + resp.media = ["test"] + +if __name__ == '__main__': + api.run() \ No newline at end of file diff --git a/api/app.py b/api/app.py index 97313da..5fa4f8f 100644 --- a/api/app.py +++ b/api/app.py @@ -2,6 +2,7 @@ from apistar import App, http, Route +from ceryx import schemas from ceryx import types from ceryx.db import RedisRouter diff --git a/api/ceryx/db.py b/api/ceryx/db.py index 679f070..650b1f0 100644 --- a/api/ceryx/db.py +++ b/api/ceryx/db.py @@ -5,6 +5,7 @@ import redis +from ceryx import schemas from ceryx import settings @@ -15,6 +16,10 @@ def _str(subject): return subject.decode("utf-8") if type(subject) == bytes else str(bytes) +def ensure_protocol(url): + return url if re.match(STARTS_WITH_PROTOCOL, url) else f"http://{url}" + + def encode_settings(settings): """ Encode and sanitize settings in order to be written to Redis. @@ -43,7 +48,7 @@ def decode_settings(settings): return decoded -class RedisRouter(object): +class RedisClient: """ Router using a redis backend, in order to route incoming requests. """ @@ -55,18 +60,15 @@ class LookupNotFound(Exception): def __init__(self, message, errors=None): Exception.__init__(self, message) - if errors is None: - self.errors = {"message": message} - else: - self.errors = errors + self.errors = errors or {"message": message} @staticmethod def from_config(path=None): """ - Returns a RedisRouter, using the default configuration from Ceryx + Returns a RedisClient, using the default configuration from Ceryx settings. """ - return RedisRouter( + return RedisClient( settings.REDIS_HOST, settings.REDIS_PORT, settings.REDIS_PASSWORD, @@ -77,41 +79,32 @@ def from_config(path=None): def __init__(self, host, port, password, db, prefix): self.client = redis.StrictRedis(host=host, port=port, password=password, db=db) self.prefix = prefix + + def _prefixed_key(self, key): + return f"{self.prefix}:{key}" - def _prefixed_route_key(self, source): + def _route_key(self, source): """ Returns the prefixed key, if prefix has been defined, for the given route. """ - prefixed_key = "routes:%s" - if self.prefix is not None: - prefixed_key = self.prefix + ":routes:%s" - prefixed_key = prefixed_key % source - return prefixed_key + return self._prefixed_key(f"routes:{source}") - def _prefixed_settings_key(self, source): + def _settings_key(self, source): """ Returns the prefixed key, if prefix has been defined, for the given source's setting. """ - prefixed_key = "settings:%s" - if self.prefix is not None: - prefixed_key = self.prefix + ":settings:%s" - prefixed_key = prefixed_key % source - return prefixed_key + return self._prefixed_key(f"settings:{source}") - def _delete_settings_for_source(self, source): - settings_key = self._prefixed_settings_key(source) + def delete_settings(self, source): + settings_key = self._settings_key(source) self.client.delete(settings_key) - def _set_settings_for_source(self, source, settings): - settings_key = self._prefixed_settings_key(source) - - if settings: - encoded_settings = encode_settings(settings) - self.client.hmset(settings_key, encoded_settings) - else: - self._delete_settings_for_source(source) + def set_settings(self, source, settings): + settings_key = self._settings_key(source) + encoded_settings = encode_settings(settings or {}) + self.client.hmset(settings_key, encoded_settings) def lookup(self, host, silent=False): """ @@ -119,11 +112,11 @@ def lookup(self, host, silent=False): the given name is found and silent is False, raises a LookupNotFound exception. """ - lookup_host = self._prefixed_route_key(host) + lookup_host = self._route_key(host) target_host = self.client.get(lookup_host) if target_host is None and not silent: - raise RedisRouter.LookupNotFound("Given host does not match with any route") + raise RedisClient.LookupNotFound("Given host does not match with any route") else: return _str(target_host) @@ -131,50 +124,44 @@ def lookup_settings(self, host): """ Fetches the settings of the given host name. """ - key = self._prefixed_settings_key(host) + key = self._settings_key(host) settings = self.client.hgetall(key) decoded_settings = decode_settings(settings) return decoded_settings - def lookup_hosts(self, pattern): + def lookup_hosts(self): """ - Fetches hosts that match the given pattern. If no pattern is given, - all hosts are returned. + Return all hosts. """ - if not pattern: - pattern = "*" - lookup_pattern = self._prefixed_route_key(pattern) + lookup_pattern = self._route_key("*") # Base + left_padding = len(lookup_pattern) - 1 keys = self.client.keys(lookup_pattern) - filtered_keys = [key[len(lookup_pattern) - len(pattern) :] for key in keys] - return [_str(key) for key in filtered_keys] - - def lookup_routes(self, pattern="*"): - """ - Fetches routes with host that matches the given pattern. If no pattern - is given, all routes are returned. - """ - hosts = self.lookup_hosts(pattern) - routes = [] - for host in hosts: - routes.append( - { - "source": host, - "target": self.lookup(host, silent=True), - "settings": self.lookup_settings(host), - } - ) + return [_str(key)[left_padding:] for key in keys] + + def get_route_for_host(self, host): + route = { + "source": host, + "target": self.lookup(host, silent=True), + "settings": self.lookup_settings(host), + } + return route + + def lookup_routes(self): + """ + Return all routes + """ + hosts = self.lookup_hosts() + routes = [self.get_route_for_host(host) for host in hosts] return routes def insert(self, source, target, settings): """ Inserts a new source/target host entry in to the database. """ - target = ( - target if re.match(STARTS_WITH_PROTOCOL, target) else f"http://{target}" - ) - route_key = self._prefixed_route_key(source) + target = ensure_protocol(target) + route_key = self._route_key(source) self.client.set(route_key, target) - self._set_settings_for_source(source, settings) + self.set_settings(source, settings) route = {"source": source, "target": target, "settings": settings} return route @@ -182,6 +169,6 @@ def delete(self, source): """ Deletes the entry of the given source, if it exists. """ - source_key = self._prefixed_route_key(source) + source_key = self._route_key(source) self.client.delete(source_key) - self._delete_settings_for_source(source) + self.delete_settings(source) diff --git a/api/ceryx/schemas.py b/api/ceryx/schemas.py new file mode 100644 index 0000000..8f33dce --- /dev/null +++ b/api/ceryx/schemas.py @@ -0,0 +1,73 @@ +import typesystem + + +def boolean_to_redis(value: bool): + return "1" if value else "0" + + +def redis_to_boolean(value): + return True if "1" else False + + +def ensure_string(value): + redis_value = ( + None if value is None + else value.decode("utf-8") if type(value) == bytes else str(value) + ) + return redis_value + + +def value_to_redis(field, value): + if isinstance(field, typesystem.Boolean): + return boolean_to_redis(value) + + if isinstance(field, typesystem.Reference): + return field.target.validate(value).to_redis() + + return ensure_string(value) + + +def redis_to_value(field, redis_value): + if isinstance(field, typesystem.Boolean): + return redis_to_boolean(redis_value) + + if isinstance(field, typesystem.Reference): + return field.target.from_redis(redis_value) + + return ensure_string(redis_value) + + +class BaseSchema(typesystem.Schema): + @classmethod + def from_redis(cls, redis_data): + return { + ensure_string(key): redis_to_value(self.fields[key], value) + for key, value in self.items() + } + + def to_redis(self): + return { + ensure_string(key): value_to_redis(self.fields[key], value) + for key, value in self.items() + } + + +class Settings(BaseSchema): + enforce_https = typesystem.Boolean(default=False) + mode = typesystem.Choice( + choices=( + ("proxy", "Proxy"), + ("redirect", "Redirect"), + ), + default="proxy", + ) + certificate_path = typesystem.String(allow_null=True) + key_path = typesystem.String(allow_null=True) + + +class Route(BaseSchema): + DEFAULT_SETTINGS = dict(Settings.validate({})) + + source = typesystem.String() + target = typesystem.String() + settings = typesystem.Reference(Settings, default=DEFAULT_SETTINGS) diff --git a/api/ceryx/types.py b/api/ceryx/types.py index 357f2a0..c637b94 100644 --- a/api/ceryx/types.py +++ b/api/ceryx/types.py @@ -5,8 +5,15 @@ properties={ "enforce_https": validators.Boolean(default=False), "mode": validators.String(default="proxy", enum=["proxy", "redirect"]), + "certificate_path": validators.String(), + "key_path": validators.String(), + }, + default={ + "enforce_https": False, + "mode": "proxy", + "certificate_path": None, + "key_path": None, }, - default={"enforce_https": False, "mode": "proxy"}, ) diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 784de65..bd6d995 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -18,6 +18,7 @@ services: environment: CERYX_API_HOSTNAME: ${CERYX_API_HOSTNAME:-api.ceryx.dev} CERYX_DEBUG: ${CERYX_DEBUG:-true} + command: uvicorn --reload --host 0.0.0.0 --port 5555 api:api networks: default: From ab33201eedc335efb827f6145ff64e0fdaa12a4f Mon Sep 17 00:00:00 2001 From: Paris Kasidiaris Date: Fri, 26 Apr 2019 11:28:14 +0300 Subject: [PATCH 14/17] Start making new API functional. --- api/api.py | 2 +- api/ceryx/db.py | 29 ++++++++++++----------------- api/ceryx/schemas.py | 7 ++++--- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/api/api.py b/api/api.py index 5138a85..b224dd0 100644 --- a/api/api.py +++ b/api/api.py @@ -7,7 +7,7 @@ @api.route("/api/routes/") async def list_routes(req, resp): - resp.media = ["test"] + resp.media = [dict(route) for route in client.list_routes()] if __name__ == '__main__': api.run() \ No newline at end of file diff --git a/api/ceryx/db.py b/api/ceryx/db.py index 650b1f0..f5e94a7 100644 --- a/api/ceryx/db.py +++ b/api/ceryx/db.py @@ -106,28 +106,21 @@ def set_settings(self, source, settings): encoded_settings = encode_settings(settings or {}) self.client.hmset(settings_key, encoded_settings) - def lookup(self, host, silent=False): + def _lookup_target(self, host): """ Fetches the target host for the given host name. If no host matching the given name is found and silent is False, raises a LookupNotFound exception. """ lookup_host = self._route_key(host) - target_host = self.client.get(lookup_host) + return self.client.get(lookup_host) - if target_host is None and not silent: - raise RedisClient.LookupNotFound("Given host does not match with any route") - else: - return _str(target_host) - - def lookup_settings(self, host): + def _lookup_settings(self, host): """ Fetches the settings of the given host name. """ key = self._settings_key(host) - settings = self.client.hgetall(key) - decoded_settings = decode_settings(settings) - return decoded_settings + return self.client.hgetall(key) def lookup_hosts(self): """ @@ -139,16 +132,18 @@ def lookup_hosts(self): return [_str(key)[left_padding:] for key in keys] def get_route_for_host(self, host): - route = { + settings = self._lookup_settings(host) + target = self._lookup_target(host) + route = schemas.Route.from_redis({ "source": host, - "target": self.lookup(host, silent=True), - "settings": self.lookup_settings(host), - } + "target": target, + "settings": settings + }) return route - def lookup_routes(self): + def list_routes(self): """ - Return all routes + Just return all routes """ hosts = self.lookup_hosts() routes = [self.get_route_for_host(host) for host in hosts] diff --git a/api/ceryx/schemas.py b/api/ceryx/schemas.py index 8f33dce..4bbd1d5 100644 --- a/api/ceryx/schemas.py +++ b/api/ceryx/schemas.py @@ -40,10 +40,11 @@ def redis_to_value(field, redis_value): class BaseSchema(typesystem.Schema): @classmethod def from_redis(cls, redis_data): - return { - ensure_string(key): redis_to_value(self.fields[key], value) - for key, value in self.items() + data = { + ensure_string(key): redis_to_value(cls.fields[ensure_string(key)], value) + for key, value in redis_data.items() } + return cls.validate(data) def to_redis(self): return { From fcad7d72adec50ba016ad3190e5bc7ad3398869d Mon Sep 17 00:00:00 2001 From: Paris Kasidiaris Date: Fri, 26 Apr 2019 17:25:39 +0300 Subject: [PATCH 15/17] Migrate to Reposponder in favor of deprecated APIStar Tests werk. --- .gitignore | 3 + api/Pipfile | 3 +- api/Pipfile.lock | 107 +++++++++++++-- api/api.py | 41 +++++- api/app.py | 106 -------------- api/bin/test | 7 +- api/ceryx/db.py | 157 +++++++-------------- api/ceryx/exceptions.py | 3 + api/ceryx/schemas.py | 15 +- api/ceryx/types.py | 28 ---- api/tests.py | 297 ++++++++++++++-------------------------- 11 files changed, 314 insertions(+), 453 deletions(-) delete mode 100644 api/app.py create mode 100644 api/ceryx/exceptions.py delete mode 100644 api/ceryx/types.py diff --git a/.gitignore b/.gitignore index 9cdf38d..3849ce1 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ docker-compose.yaml # Ignore generated files from template ceryx/nginx/conf/ceryx.conf ceryx/nginx/conf/nginx.conf + +.mypy_cache +.pytest_cache \ No newline at end of file diff --git a/api/Pipfile b/api/Pipfile index 8b0a525..5b6bf78 100644 --- a/api/Pipfile +++ b/api/Pipfile @@ -4,7 +4,6 @@ verify_ssl = true name = "pypi" [packages] -apistar = "==0.4.3" redis = "*" requests = ">=2.21.0" typesystem = "*" @@ -13,6 +12,8 @@ responder = "*" [dev-packages] nose = "*" black = "==18.9b0" +pytest = "*" +mypy = "*" [requires] python_version = "3.6" diff --git a/api/Pipfile.lock b/api/Pipfile.lock index 8f833bd..409504e 100644 --- a/api/Pipfile.lock +++ b/api/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "c96798be36a81423cba68109512b6b8cb61e43e6ba62c6626eb8cf87ba8dc9b5" + "sha256": "79eaad70504a24b4d81cc5b1479a8c3b77104b140781e05dfeb83608a6fc7fa7" }, "pipfile-spec": 6, "requires": { @@ -39,10 +39,9 @@ }, "apistar": { "hashes": [ - "sha256:e4c82c8c1467a4a76ddf431b65686aebe89f37cefeb320dc8dcebfeb5928ab20" + "sha256:8da0d3f15748c8ed6e68914ba5b8f6dd5dff5afbe137950d07103575df0bce73" ], - "index": "pypi", - "version": "==0.4.3" + "version": "==0.7.2" }, "asgiref": { "hashes": [ @@ -339,13 +338,6 @@ ], "version": "==7.0" }, - "werkzeug": { - "hashes": [ - "sha256:0a73e8bb2ff2feecfc5d56e6f458f5b99290ef34f565ffb2665801ff7de6af7a", - "sha256:7fad9770a8778f9576693f0cc29c7dcc36964df916b83734f4431c0e612a7fbc" - ], - "version": "==0.15.2" - }, "whitenoise": { "hashes": [ "sha256:118ab3e5f815d380171b100b05b76de2a07612f422368a201a9ffdeefb2251c1", @@ -362,6 +354,13 @@ ], "version": "==1.4.3" }, + "atomicwrites": { + "hashes": [ + "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", + "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" + ], + "version": "==1.3.0" + }, "attrs": { "hashes": [ "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", @@ -384,6 +383,38 @@ ], "version": "==7.0" }, + "more-itertools": { + "hashes": [ + "sha256:2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7", + "sha256:c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a" + ], + "markers": "python_version > '2.7'", + "version": "==7.0.0" + }, + "mypy": { + "hashes": [ + "sha256:2afe51527b1f6cdc4a5f34fc90473109b22bf7f21086ba3e9451857cf11489e6", + "sha256:56a16df3e0abb145d8accd5dbb70eba6c4bd26e2f89042b491faa78c9635d1e2", + "sha256:5764f10d27b2e93c84f70af5778941b8f4aa1379b2430f85c827e0f5464e8714", + "sha256:5bbc86374f04a3aa817622f98e40375ccb28c4836f36b66706cf3c6ccce86eda", + "sha256:6a9343089f6377e71e20ca734cd8e7ac25d36478a9df580efabfe9059819bf82", + "sha256:6c9851bc4a23dc1d854d3f5dfd5f20a016f8da86bcdbb42687879bb5f86434b0", + "sha256:b8e85956af3fcf043d6f87c91cbe8705073fc67029ba6e22d3468bfee42c4823", + "sha256:b9a0af8fae490306bc112229000aa0c2ccc837b49d29a5c42e088c132a2334dd", + "sha256:bbf643528e2a55df2c1587008d6e3bda5c0445f1240dfa85129af22ae16d7a9a", + "sha256:c46ab3438bd21511db0f2c612d89d8344154c0c9494afc7fbc932de514cf8d15", + "sha256:f7a83d6bd805855ef83ec605eb01ab4fa42bcef254b13631e451cbb44914a9b0" + ], + "index": "pypi", + "version": "==0.701" + }, + "mypy-extensions": { + "hashes": [ + "sha256:37e0e956f41369209a3d5f34580150bcacfabaa57b33a15c0b25f4b5725e0812", + "sha256:b16cabe759f55e3409a7d231ebd2841378fb0c27a5d1994719e340e4f429ac3e" + ], + "version": "==0.4.1" + }, "nose": { "hashes": [ "sha256:9ff7c6cc443f8c51994b34a667bbcf45afd6d945be7477b52e97516fd17c53ac", @@ -393,6 +424,35 @@ "index": "pypi", "version": "==1.3.7" }, + "pluggy": { + "hashes": [ + "sha256:19ecf9ce9db2fce065a7a0586e07cfb4ac8614fe96edf628a264b1c70116cf8f", + "sha256:84d306a647cc805219916e62aab89caa97a33a1dd8c342e87a37f91073cd4746" + ], + "version": "==0.9.0" + }, + "py": { + "hashes": [ + "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", + "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" + ], + "version": "==1.8.0" + }, + "pytest": { + "hashes": [ + "sha256:3773f4c235918987d51daf1db66d51c99fac654c81d6f2f709a046ab446d5e5d", + "sha256:b7802283b70ca24d7119b32915efa7c409982f59913c1a6c0640aacf118b95f5" + ], + "index": "pypi", + "version": "==4.4.1" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, "toml": { "hashes": [ "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", @@ -400,6 +460,31 @@ "sha256:f1db651f9657708513243e61e6cc67d101a39bad662eaa9b5546f789338e07a3" ], "version": "==0.10.0" + }, + "typed-ast": { + "hashes": [ + "sha256:04894d268ba6eab7e093d43107869ad49e7b5ef40d1a94243ea49b352061b200", + "sha256:16616ece19daddc586e499a3d2f560302c11f122b9c692bc216e821ae32aa0d0", + "sha256:252fdae740964b2d3cdfb3f84dcb4d6247a48a6abe2579e8029ab3be3cdc026c", + "sha256:2af80a373af123d0b9f44941a46df67ef0ff7a60f95872412a145f4500a7fc99", + "sha256:2c88d0a913229a06282b285f42a31e063c3bf9071ff65c5ea4c12acb6977c6a7", + "sha256:2ea99c029ebd4b5a308d915cc7fb95b8e1201d60b065450d5d26deb65d3f2bc1", + "sha256:3d2e3ab175fc097d2a51c7a0d3fda442f35ebcc93bb1d7bd9b95ad893e44c04d", + "sha256:4766dd695548a15ee766927bf883fb90c6ac8321be5a60c141f18628fb7f8da8", + "sha256:56b6978798502ef66625a2e0f80cf923da64e328da8bbe16c1ff928c70c873de", + "sha256:5cddb6f8bce14325b2863f9d5ac5c51e07b71b462361fd815d1d7706d3a9d682", + "sha256:644ee788222d81555af543b70a1098f2025db38eaa99226f3a75a6854924d4db", + "sha256:64cf762049fc4775efe6b27161467e76d0ba145862802a65eefc8879086fc6f8", + "sha256:68c362848d9fb71d3c3e5f43c09974a0ae319144634e7a47db62f0f2a54a7fa7", + "sha256:6c1f3c6f6635e611d58e467bf4371883568f0de9ccc4606f17048142dec14a1f", + "sha256:b213d4a02eec4ddf622f4d2fbc539f062af3788d1f332f028a2e19c42da53f15", + "sha256:bb27d4e7805a7de0e35bd0cb1411bc85f807968b2b0539597a49a23b00a622ae", + "sha256:c9d414512eaa417aadae7758bc118868cd2396b0e6138c1dd4fda96679c079d3", + "sha256:f0937165d1e25477b01081c4763d2d9cdc3b18af69cb259dd4f640c9b900fe5e", + "sha256:fb96a6e2c11059ecf84e6741a319f93f683e440e341d4489c9b161eca251cf2a", + "sha256:fc71d2d6ae56a091a8d94f33ec9d0f2001d1cb1db423d8b4355debfe9ce689b7" + ], + "version": "==1.3.4" } } } diff --git a/api/api.py b/api/api.py index b224dd0..f404a45 100644 --- a/api/api.py +++ b/api/api.py @@ -1,13 +1,50 @@ import responder from ceryx.db import RedisClient +from ceryx.exceptions import NotFound + api = responder.API() client = RedisClient.from_config() + +@api.route(default=True) +def default(req, resp): + if not req.url.path.endswith("/"): + api.redirect(resp, f"{req.url.path}/") + + @api.route("/api/routes/") -async def list_routes(req, resp): - resp.media = [dict(route) for route in client.list_routes()] +class RouteListView: + async def on_get(self, req, resp): + resp.media = [dict(route) for route in client.list_routes()] + + async def on_post(self, req, resp): + data = await req.media() + route = client.create_route(data) + resp.status_code = api.status_codes.HTTP_201 + resp.media = dict(route) + + +@api.route("/api/routes/{host}/") +class RouteDetailView: + async def on_get(self, req, resp, *, host: str): + try: + route = client.get_route(host) + resp.media = dict(route) + except NotFound: + resp.media = {"detail": f"No route found for {host}."} + resp.status_code = 404 + + async def on_put(self, req, resp, *, host: str): + data = await req.media() + route = client.update_route(host, data) + resp.media = dict(route) + + async def on_delete(self, req, resp, *, host:str): + client.delete_route(host) + resp.status_code = api.status_codes.HTTP_204 + if __name__ == '__main__': api.run() \ No newline at end of file diff --git a/api/app.py b/api/app.py deleted file mode 100644 index 5fa4f8f..0000000 --- a/api/app.py +++ /dev/null @@ -1,106 +0,0 @@ -import typing - -from apistar import App, http, Route - -from ceryx import schemas -from ceryx import types -from ceryx.db import RedisRouter - - -ROUTER = RedisRouter.from_config() - - -def list_routes() -> typing.List[types.Route]: - routes = ROUTER.lookup_routes() - return [types.Route(route) for route in routes] - - -def create_route(route: types.Route) -> types.Route: - created_route = ROUTER.insert(**route) - return http.JSONResponse(created_route, status_code=201) - - -def update_route(source: str, route: types.RouteWithoutSource) -> types.Route: - updated_route = ROUTER.insert(source, **route) - return types.Route(updated_route) - - -def get_route(source: str) -> types.Route: - try: - resource = { - "source": source, - "target": ROUTER.lookup(source), - "settings": ROUTER.lookup_settings(source), - } - return resource - except RedisRouter.LookupNotFound: - return http.JSONResponse( - {"message": f"Route with source {source} doesn't exist"}, status_code=404 - ) - - -def delete_route(source: str) -> types.Route: - try: - route = { - "source": source, - "target": ROUTER.lookup(source), - "settings": ROUTER.lookup_settings(source), - } - ROUTER.delete(source) - return http.JSONResponse(types.Route(route), status_code=204) - except RedisRouter.LookupNotFound: - return http.JSONResponse( - {"message": f"Route with source {source} doesn't exist"}, status_code=404 - ) - - -routes = [ - Route("/api/routes", method="GET", handler=list_routes), - Route("/api/routes", method="POST", handler=create_route), - Route("/api/routes/{source}", method="GET", handler=get_route), - Route("/api/routes/{source}", method="PUT", handler=update_route), - Route("/api/routes/{source}", method="DELETE", handler=delete_route), - # Allow trailing slashes as well (GitHub style) - Route( - "/api/routes/", - method="GET", - handler=list_routes, - name="list_routes_trailing_slash", - ), - Route( - "/api/routes/", - method="POST", - handler=create_route, - name="create_route_trailing_slash", - ), - Route( - "/api/routes/{source}/", - method="GET", - handler=get_route, - name="get_route_trailing_slash", - ), - Route( - "/api/routes/{source}/", - method="PUT", - handler=update_route, - name="update_route_trailing_slash", - ), - Route( - "/api/routes/{source}/", - method="DELETE", - handler=delete_route, - name="delete_route_trailing_slash", - ), -] - -app = App(routes=routes) - -if __name__ == "__main__": - from ceryx import settings - - app.serve( - settings.API_BIND_HOST, - settings.API_BIND_PORT, - use_debugger=settings.DEBUG, - use_reloader=settings.DEBUG, - ) diff --git a/api/bin/test b/api/bin/test index bd23b15..b1a4e57 100755 --- a/api/bin/test +++ b/api/bin/test @@ -2,8 +2,7 @@ # # Run the Ceryx API test suite, using Nose. -set -e +set -ex -export CERYX_REDIS_PREFIX=ceryxtests - -nosetests +mypy --ignore-missing-imports api.py +pytest tests.py diff --git a/api/ceryx/db.py b/api/ceryx/db.py index f5e94a7..052894b 100644 --- a/api/ceryx/db.py +++ b/api/ceryx/db.py @@ -1,67 +1,16 @@ """ Simple Redis client, implemented the data logic of Ceryx. """ -import re - import redis -from ceryx import schemas -from ceryx import settings - - -STARTS_WITH_PROTOCOL = r"^https?://" +from ceryx import exceptions, schemas, settings def _str(subject): return subject.decode("utf-8") if type(subject) == bytes else str(bytes) -def ensure_protocol(url): - return url if re.match(STARTS_WITH_PROTOCOL, url) else f"http://{url}" - - -def encode_settings(settings): - """ - Encode and sanitize settings in order to be written to Redis. - """ - encoded_settings = { - "enforce_https": str(int(settings.get("enforce_https", False))), - "mode": settings.get("mode", "proxy"), - } - - return encoded_settings - - -def decode_settings(settings): - """ - Decode and sanitize settings from Redis, in order to transport via HTTP - """ - - # If any of the keys or values of the provided settings are bytes, then - # convert them to strings. - _settings = {_str(k): _str(v) for k, v in settings.items()} - decoded = { - "enforce_https": bool(int(_settings.get("enforce_https", "0"))), - "mode": _settings.get("mode", "proxy"), - } - - return decoded - - class RedisClient: - """ - Router using a redis backend, in order to route incoming requests. - """ - - class LookupNotFound(Exception): - """ - Exception raised when a lookup for a specific host was not found. - """ - - def __init__(self, message, errors=None): - Exception.__init__(self, message) - self.errors = errors or {"message": message} - @staticmethod def from_config(path=None): """ @@ -84,56 +33,55 @@ def _prefixed_key(self, key): return f"{self.prefix}:{key}" def _route_key(self, source): - """ - Returns the prefixed key, if prefix has been defined, for the given - route. - """ return self._prefixed_key(f"routes:{source}") def _settings_key(self, source): - """ - Returns the prefixed key, if prefix has been defined, for the given - source's setting. - """ return self._prefixed_key(f"settings:{source}") - def delete_settings(self, source): - settings_key = self._settings_key(source) - self.client.delete(settings_key) - - def set_settings(self, source, settings): - settings_key = self._settings_key(source) - encoded_settings = encode_settings(settings or {}) - self.client.hmset(settings_key, encoded_settings) + def _delete_target(self, host): + key = self._route_key(host) + self.client.delete(key) + + def _delete_settings(self, host): + key = self._settings_key(host) + self.client.delete(key) - def _lookup_target(self, host): - """ - Fetches the target host for the given host name. If no host matching - the given name is found and silent is False, raises a LookupNotFound - exception. - """ - lookup_host = self._route_key(host) - return self.client.get(lookup_host) + def _lookup_target(self, host, raise_exception=False): + key = self._route_key(host) + target = self.client.get(key) + + if target is None and raise_exception: + raise exceptions.NotFound("Route not found.") + + return target def _lookup_settings(self, host): - """ - Fetches the settings of the given host name. - """ key = self._settings_key(host) return self.client.hgetall(key) - def lookup_hosts(self): - """ - Return all hosts. - """ - lookup_pattern = self._route_key("*") # Base + def lookup_hosts(self, pattern="*"): + lookup_pattern = self._route_key(pattern) left_padding = len(lookup_pattern) - 1 keys = self.client.keys(lookup_pattern) return [_str(key)[left_padding:] for key in keys] - def get_route_for_host(self, host): + def _set_target(self, host, target): + key = self._route_key(host) + self.client.set(key, target) + + def _set_settings(self, host, settings): + key = self._settings_key(host) + self.client.hmset(key, settings) + + def _set_route(self, route: schemas.Route): + redis_data = route.to_redis() + self._set_target(route.source, redis_data["target"]) + self._set_settings(route.source, redis_data["settings"]) + return route + + def get_route(self, host): + target = self._lookup_target(host, raise_exception=True) settings = self._lookup_settings(host) - target = self._lookup_target(host) route = schemas.Route.from_redis({ "source": host, "target": target, @@ -142,28 +90,19 @@ def get_route_for_host(self, host): return route def list_routes(self): - """ - Just return all routes - """ hosts = self.lookup_hosts() - routes = [self.get_route_for_host(host) for host in hosts] + routes = [self.get_route(host) for host in hosts] return routes - - def insert(self, source, target, settings): - """ - Inserts a new source/target host entry in to the database. - """ - target = ensure_protocol(target) - route_key = self._route_key(source) - self.client.set(route_key, target) - self.set_settings(source, settings) - route = {"source": source, "target": target, "settings": settings} - return route - - def delete(self, source): - """ - Deletes the entry of the given source, if it exists. - """ - source_key = self._route_key(source) - self.client.delete(source_key) - self.delete_settings(source) + + def create_route(self, data: dict): + route = schemas.Route.validate(data) + return self._set_route(route) + + def update_route(self, host: str, data: dict): + data["source"] = host + route = schemas.Route.validate(data) + return self._set_route(route) + + def delete_route(self, host: str): + self._delete_target(host) + self._delete_settings(host) diff --git a/api/ceryx/exceptions.py b/api/ceryx/exceptions.py new file mode 100644 index 0000000..3c3c444 --- /dev/null +++ b/api/ceryx/exceptions.py @@ -0,0 +1,3 @@ +class NotFound(Exception): + status_code = 404 + pass \ No newline at end of file diff --git a/api/ceryx/schemas.py b/api/ceryx/schemas.py index 4bbd1d5..b1dfbc6 100644 --- a/api/ceryx/schemas.py +++ b/api/ceryx/schemas.py @@ -1,12 +1,18 @@ +import re import typesystem +def ensure_protocol(url): + starts_with_protocol = r"^https?://" + return url if re.match(starts_with_protocol, url) else f"http://{url}" + + def boolean_to_redis(value: bool): return "1" if value else "0" def redis_to_boolean(value): - return True if "1" else False + return True if value == "1" else False def ensure_string(value): @@ -50,6 +56,7 @@ def to_redis(self): return { ensure_string(key): value_to_redis(self.fields[key], value) for key, value in self.items() + if value is not None } @@ -72,3 +79,9 @@ class Route(BaseSchema): source = typesystem.String() target = typesystem.String() settings = typesystem.Reference(Settings, default=DEFAULT_SETTINGS) + + @classmethod + def validate(cls, data): + if "target" in data.keys(): + data["target"] = ensure_protocol(data["target"]) + return super().validate(data) diff --git a/api/ceryx/types.py b/api/ceryx/types.py deleted file mode 100644 index c637b94..0000000 --- a/api/ceryx/types.py +++ /dev/null @@ -1,28 +0,0 @@ -from apistar import types, validators - - -SETTINGS_VALIDATOR = validators.Object( - properties={ - "enforce_https": validators.Boolean(default=False), - "mode": validators.String(default="proxy", enum=["proxy", "redirect"]), - "certificate_path": validators.String(), - "key_path": validators.String(), - }, - default={ - "enforce_https": False, - "mode": "proxy", - "certificate_path": None, - "key_path": None, - }, -) - - -class RouteWithoutSource(types.Type): - target = validators.String() - settings = SETTINGS_VALIDATOR - - -class Route(types.Type): - source = validators.String() - target = validators.String() - settings = SETTINGS_VALIDATOR diff --git a/api/tests.py b/api/tests.py index c828d18..f34e7a7 100644 --- a/api/tests.py +++ b/api/tests.py @@ -1,191 +1,106 @@ -import unittest - -from apistar import test - -from app import app - - -CLIENT = test.TestClient(app) - - -class CeryxTestCase(unittest.TestCase): - def setUp(self): - self.client = CLIENT - - def test_list_routes(self): - """ - Assert that listing routes will return a JSON list. - """ - response = self.client.get("/api/routes") - self.assertEqual(response.status_code, 200) - self.assertEqual(type(response.json()), list) - - def test_create_route_without_protocol(self): - """ - Assert that creating a route, will result in the appropriate route. - """ - request_body = {"source": "test.dev", "target": "localhost:11235"} - response_body = { - "source": "test.dev", - "target": "http://localhost:11235", - "settings": {"enforce_https": False, "mode": "proxy"}, - } - - # Create a route and assert valid data in response - response = self.client.post("/api/routes", json=request_body) - self.assertEqual(response.status_code, 201) - self.assertDictEqual(response.json(), response_body) - - # Also get the route and assert valid data - response = self.client.get("/api/routes/test.dev") - self.assertDictEqual(response.json(), response_body) - - def test_create_route_with_http_protocol(self): - """ - Assert that creating a route, will result in the appropriate route. - """ - request_body = {"source": "test.dev", "target": "http://localhost:11235"} - response_body = { - "source": "test.dev", - "target": "http://localhost:11235", - "settings": {"enforce_https": False, "mode": "proxy"}, - } - - # Create a route and assert valid data in response - response = self.client.post("/api/routes", json=request_body) - self.assertEqual(response.status_code, 201) - self.assertDictEqual(response.json(), response_body) - - # Also get the route and assert valid data - response = self.client.get("/api/routes/test.dev") - self.assertDictEqual(response.json(), response_body) - - def test_create_route_with_https_protocol(self): - """ - Assert that creating a route, will result in the appropriate route. - """ - request_body = {"source": "test.dev", "target": "https://localhost:11235"} - response_body = { - "source": "test.dev", - "target": "https://localhost:11235", - "settings": {"enforce_https": False, "mode": "proxy"}, - } - - # Create a route and assert valid data in response - response = self.client.post("/api/routes", json=request_body) - self.assertEqual(response.status_code, 201) - self.assertDictEqual(response.json(), response_body) - - # Also get the route and assert valid data - response = self.client.get("/api/routes/test.dev") - self.assertDictEqual(response.json(), response_body) - - def test_enforce_https(self): - """ - Assert that creating a route with the `enforce_https` settings returns - the expected results - """ - route_without_enforce_https_request_body = { - "source": "test-no-enforce-https.dev", - "target": "http://localhost:11235", - } - route_enforce_https_true = { - "source": "test-enforce-https-true.dev", - "target": "http://localhost:11235", - "settings": {"enforce_https": True, "mode": "proxy"}, - } - route_enforce_https_false = { - "source": "test-enforce-https-false.dev", - "target": "http://localhost:11235", - "settings": {"enforce_https": False, "mode": "proxy"}, - } - route_without_enforce_https_response_body = { - "source": "test-no-enforce-https.dev", - "target": "http://localhost:11235", - "settings": {"enforce_https": False, "mode": "proxy"}, - } - - response = self.client.post( - "/api/routes", json=route_without_enforce_https_request_body - ) - self.assertEqual(response.status_code, 201) - self.assertDictEqual(response.json(), route_without_enforce_https_response_body) - - response = self.client.get("/api/routes/test-no-enforce-https.dev") - self.assertDictEqual(response.json(), route_without_enforce_https_response_body) - - response = self.client.post("/api/routes", json=route_enforce_https_true) - self.assertEqual(response.status_code, 201) - self.assertDictEqual(response.json(), route_enforce_https_true) - - response = self.client.get("/api/routes/test-enforce-https-true.dev") - self.assertDictEqual(response.json(), route_enforce_https_true) - - response = self.client.post("/api/routes", json=route_enforce_https_false) - self.assertEqual(response.status_code, 201) - self.assertDictEqual(response.json(), route_enforce_https_false) - - response = self.client.get("/api/routes/test-enforce-https-false.dev") - self.assertDictEqual(response.json(), route_enforce_https_false) - - def test_mode(self): - """ - Assert that creating a route with or without the `mode` setting returns - the expected results. - """ - route_without_mode = { - "source": "www.my-website.dev", - "target": "http://localhost:11235", - } - route_mode_proxy = { - "source": "www.my-website.dev", - "target": "http://localhost:11235", - "settings": {"enforce_https": False, "mode": "proxy"}, - } - route_mode_redirect = { - "source": "my-website.dev", - "target": "http://www.my-website.dev", - "settings": {"enforce_https": False, "mode": "redirect"}, - } - - response = self.client.post("/api/routes", json=route_without_mode) - self.assertEqual(response.status_code, 201) - self.assertDictEqual(response.json(), route_mode_proxy) - - response = self.client.get("/api/routes/www.my-website.dev") - self.assertDictEqual(response.json(), route_mode_proxy) - - response = self.client.post("/api/routes", json=route_mode_proxy) - self.assertEqual(response.status_code, 201) - self.assertDictEqual(response.json(), route_mode_proxy) - - response = self.client.get("/api/routes/www.my-website.dev") - self.assertDictEqual(response.json(), route_mode_proxy) - - response = self.client.post("/api/routes", json=route_mode_redirect) - self.assertEqual(response.status_code, 201) - self.assertDictEqual(response.json(), route_mode_redirect) - - response = self.client.get("/api/routes/my-website.dev") - self.assertDictEqual(response.json(), route_mode_redirect) - - def test_delete_route(self): - """ - Assert that deleting a route, will actually delete it. - """ - route_data = {"source": "test.dev", "target": "http://localhost:11235"} - - # Create a route - response = self.client.post("/api/routes", json=route_data) - - # Delete the route - response = self.client.delete("/api/routes/test.dev") - self.assertEqual(response.status_code, 204) - - # Also get the route and assert that it does not exist - response = self.client.get("/api/routes/test.dev") - self.assertEqual(response.status_code, 404) - - -if __name__ == "__main__": - unittest.main() +import uuid + +import pytest + +from api import api +from ceryx import schemas + + +@pytest.fixture +def client(): + return api.requests + + +@pytest.fixture +def host(): + return f"{uuid.uuid4()}.api.ceryx.test" + + +def test_list_routes(client, host): + """ + Assert that listing routes will return a JSON list. + """ + route_1 = schemas.Route.validate({ + "source": f"route-1-{host}", + "target": "http://somewhere", + }) + client.post("/api/routes/", json=dict(route_1)) + + route_2 = schemas.Route.validate({ + "source": f"route-2-{host}", + "target": "http://somewhere", + }) + client.post("/api/routes/", json=dict(route_2)) + + response = client.get("/api/routes/") + assert response.status_code == 200 + + route_list = response.json() + assert dict(route_1) in route_list + assert dict(route_2) in route_list + + +def test_create_route(client, host): + """ + Assert that creating a route, will result in the appropriate route. + """ + route = schemas.Route.validate({ + "source": host, + "target": "http://somewhere", + }) + + # Create a route and assert valid data in response + response = client.post("/api/routes/", json=dict(route)) + assert response.status_code == 201 + assert response.json() == dict(route) + + # Also get the route and assert valid data + response = client.get(f"/api/routes/{host}/") + assert response.status_code == 200 + assert response.json() == dict(route) + + +def test_update_route(client, host): + """ + Assert that creating a route, will result in the appropriate route. + """ + route = schemas.Route.validate({ + "source": host, + "target": "http://somewhere", + }) + + client.post("/api/routes/", json=dict(route)) + + updated_route = schemas.Route.validate({ + "source": host, + "target": "http://somewhere-else", + }) + updated_route_payload = dict(updated_route) + del updated_route_payload["source"] # We should not need that + response = client.put(f"/api/routes/{host}/", json=updated_route_payload) + + # Also get the route and assert valid data + assert response.status_code == 200 + assert response.json() == dict(updated_route) + + +def test_delete_route(client, host): + """ + Assert that deleting a route, will actually delete it. + """ + route = schemas.Route.validate({ + "source": host, + "target": "http://somewhere", + }) + + # Create a route + client.post("/api/routes/", json=dict(route)) + + # Delete the route + response = client.delete(f"/api/routes/{host}/") + assert response.status_code == 204 + + # Also get the route and assert that it does not exist + response = client.get(f"/api/routes/{host}/") + assert response.status_code == 404 + From cf4aadf7f4a9b81d021c1bbf2f5f01b553b67eaf Mon Sep 17 00:00:00 2001 From: Paris Kasidiaris Date: Fri, 26 Apr 2019 17:40:26 +0300 Subject: [PATCH 16/17] Improve CI tests --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 517f33c..a20d354 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,14 +10,12 @@ install: - pip install --upgrade --ignore-installed docker-compose==${DOCKER_COMPOSE_VERSION} - docker-compose build - - pip install pipenv==2018.11.26 - - bash -c "cd api && pipenv install --dev --deploy --system" - services: - redis-server - docker script: + - docker-compose up -d - docker-compose run api ./bin/test - docker-compose run test From 2ed1f4bcbedacf56e4b7de61ddf901f98a895f52 Mon Sep 17 00:00:00 2001 From: Paris Kasidiaris Date: Fri, 26 Apr 2019 17:59:58 +0300 Subject: [PATCH 17/17] Fix Ceryx tests --- ceryx/tests/base.py | 2 + ceryx/tests/test_certificates.py | 9 ++--- ceryx/tests/test_routes.py | 64 ++++++++------------------------ 3 files changed, 20 insertions(+), 55 deletions(-) diff --git a/ceryx/tests/base.py b/ceryx/tests/base.py index cb2f8af..eaad0d2 100644 --- a/ceryx/tests/base.py +++ b/ceryx/tests/base.py @@ -10,5 +10,7 @@ class BaseTest: def setup_method(self): self.uuid = uuid.uuid4() self.host = f"{self.uuid}.ceryx.test" + self.redis_target_key = f"ceryx:routes:{self.host}" + self.redis_settings_key = f"ceryx:settings:{self.host}" self.client = CeryxTestClient() self.redis = redis.Redis(host='redis') \ No newline at end of file diff --git a/ceryx/tests/test_certificates.py b/ceryx/tests/test_certificates.py index 8e7d836..75fd2ff 100644 --- a/ceryx/tests/test_certificates.py +++ b/ceryx/tests/test_certificates.py @@ -11,12 +11,9 @@ def test_custom_certificate(self): certificate_path , key_path = create_certificates_for_host(self.host) api_base_url = "http://api:5555/" + self.redis.set(self.redis_target_key, api_base_url) - route_redis_key = f"ceryx:routes:{self.host}" - self.redis.set(route_redis_key, api_base_url) - - settings_redis_key = f"ceryx:settings:{self.host}" - self.redis.hset(settings_redis_key, "certificate_path", certificate_path) - self.redis.hset(settings_redis_key, "key_path", key_path) + self.redis.hset(self.redis_settings_key, "certificate_path", certificate_path) + self.redis.hset(self.redis_settings_key, "key_path", key_path) self.client.get(f"https://{self.host}/", verify=certificate_path) diff --git a/ceryx/tests/test_routes.py b/ceryx/tests/test_routes.py index 19982c4..1688e56 100644 --- a/ceryx/tests/test_routes.py +++ b/ceryx/tests/test_routes.py @@ -1,16 +1,6 @@ -import os - -import requests - from base import BaseTest -CERYX_API_URL = os.getenv("CERYX_API_URL", "http://api:5555") -CERYX_API_ROUTES_ROOT = os.path.join(CERYX_API_URL, "api/routes") - -CERYX_HOST = "http://ceryx" - - class TestRoutes(BaseTest): def test_no_route(self): """ @@ -26,18 +16,12 @@ def test_proxy(self): Ceryx should successfully proxy the upstream request to the client, for a registered route. """ - api_upstream_host = "api" - ceryx_route_source = "api.ceryx.test" - ceryx_route_target = f"http://{api_upstream_host}:5555/api/routes" - # Register the local Ceryx API as a route - register_api_response = requests.post( - CERYX_API_ROUTES_ROOT, - json={"source": ceryx_route_source, "target": ceryx_route_target}, - ) + target = f"http://api:5555/api/routes/" + self.redis.set(self.redis_target_key, target) - upstream_response = self.client.get(ceryx_route_target) - ceryx_response = self.client.get(f"http://{ceryx_route_source}/") + upstream_response = self.client.get(target) + ceryx_response = self.client.get(f"http://{self.host}/") assert upstream_response.status_code == ceryx_response.status_code assert upstream_response.content == ceryx_response.content @@ -48,22 +32,13 @@ def test_redirect(self): Ceryx should respond with 301 status and the appropriate `Location` header for redirected routes. """ - api_upstream_host = "api" - ceryx_route_target = "http://api:5555/api/routes" - ceryx_route_source = "redirected-api.ceryx.test" - - # Register the local Ceryx API as a route - register_api_response = requests.post( - CERYX_API_ROUTES_ROOT, - json={ - "source": ceryx_route_source, - "target": ceryx_route_target, - "settings": {"mode": "redirect"}, - }, - ) + # Register the local Ceryx API as a redirect route + target = "http://api:5555/api/routes" + self.redis.set(self.redis_target_key, target) + self.redis.hset(self.redis_settings_key, "mode", "redirect") - url = f"http://{ceryx_route_source}/some/path/?some=args&more=args" - target_url = f"{ceryx_route_target}/some/path/?some=args&more=args" + url = f"http://{self.host}/some/path/?some=args&more=args" + target_url = f"{target}/some/path/?some=args&more=args" ceryx_response = self.client.get(url, allow_redirects=False) @@ -76,21 +51,12 @@ def test_enforce_https(self): Ceryx should respond with 301 status and the appropriate `Location` header for routes with HTTPS enforced. """ - api_upstream_host = "api" - api_upstream_target = "http://api:5555/" - ceryx_route_source = "secure-api.ceryx.test" - - # Register the local Ceryx API as a route - register_api_response = requests.post( - CERYX_API_ROUTES_ROOT, - json={ - "source": ceryx_route_source, - "target": api_upstream_target, - "settings": {"enforce_https": True}, - }, - ) + # Register the local Ceryx API as a redirect route + target = "http://api:5555/" + self.redis.set(self.redis_target_key, target) + self.redis.hset(self.redis_settings_key, "enforce_https", "1") - base_url = f"{ceryx_route_source}/some/path/?some=args&more=args" + base_url = f"{self.host}/some/path/?some=args&more=args" http_url = f"http://{base_url}" https_url = f"https://{base_url}" ceryx_response = self.client.get(http_url, allow_redirects=False)