diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..62a7a1f --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,38 @@ +name: Docker Image CI (amd64) + +on: + push: + branches: [ "main" ] + +jobs: + build-and-publish: + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + platforms: linux/amd64 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image (amd64) + uses: docker/build-push-action@v3 + with: + context: . + push: true + platforms: linux/amd64 + tags: | + ghcr.io/${{ github.repository }}:${{ github.run_number }} + ghcr.io/${{ github.repository }}:latest diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..406dbb4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,54 @@ +FROM ubuntu:24.04 + +# hadolint ignore=DL3008 +RUN mkdir -p /code && \ + sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources && \ + apt-get update && apt-get install -y --no-install-recommends \ + git && \ + apt-get update && apt-get build-dep -y \ + libssl-dev \ + curl \ + && rm -rf /var/apt/lists/* + +WORKDIR /code +# hadolint ignore=DL3003,DL3013,SC1091 +RUN git clone --depth=1 https://github.com/defo-project/openssl openssl && \ + cd openssl && \ + ./config && \ + make -j8 && \ + make install_sw && \ + ldconfig && \ + cd .. && rm -rf openssl && \ + git clone --depth=1 https://github.com/defo-project/curl && \ + cd curl && \ + autoreconf -fi && \ + PKG_CONFIG_PATH=/usr/local/lib/pkgconfig \ + ./configure --with-openssl \ + --enable-ech --enable-httpsrr \ + --enable-debug && \ + make -j8 && \ + make install && \ + cd .. && rm -rf curl && \ + git clone --depth=1 https://github.com/irl/cpython.git && \ + cd cpython && \ + git checkout ech && \ + LD_LIBRARY_PATH=/usr/local/lib \ + LDFLAGS="-Wl,-rpath,/usr/local/lib" \ + CFLAGS="-I/usr/local/include" \ + ./configure --with-openssl=/usr/local && \ + make -j8 && \ + make install && \ + cd .. && rm -rf cpython && \ + mkdir -p /code/test-code && \ + LD_LIBRARY_PATH=/usr/local/lib \ + /usr/local/bin/python3.13 -m venv /code/venv && \ + . /code/venv/bin/activate && \ + pip install --no-cache-dir certifi dnspython httptools + +WORKDIR /code/test-code +COPY ./run_command.sh /code/test-code/run_command.sh +COPY ./pyclient.py /code/test-code/pyclient.py +COPY ./targets.json /code/test-code/targets.json +RUN chmod +x /code/test-code/run_command.sh + +ENTRYPOINT ["/code/test-code/run_command.sh"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a624855 --- /dev/null +++ b/Makefile @@ -0,0 +1,3 @@ + +lint: + docker run --rm -i hadolint/hadolint < Dockerfile diff --git a/README.md b/README.md new file mode 100644 index 0000000..35dc5a1 --- /dev/null +++ b/README.md @@ -0,0 +1,33 @@ +# DEfO Client Docker Image + +This Docker image provides a simple way to use some of the experimental TLS ECH-capable clients. + +## Usage + +You can fetch the latest image from the GitHub Container Registry using the following command: + +```bash +docker pull ghcr.io/defo-project/docker-defo-client:latest +``` + +The image includes an entrypoint script that you can use to run different commands: `run_command.sh`. + +After fetching the image, you can run the container and pass commands as arguments, like this: + +### 1. Run Curl + +```bash +docker run --rm ghcr.io/defo-project/docker-defo-client:latest curl --doh-url https://1.1.1.1/dns-query --ech true https://test.defo.ie/ +``` + +### 2. Run pyclient + +```bash +docker run --rm ghcr.io/defo-project/docker-defo-client:latest pyclient -v get https://defo.ie/ech-check.php +``` + +Or to fetch a number of test endpoints: + +```bash +docker run --rm ghcr.io/defo-project/docker-defo-client:latest pyclient -v getlist --demo +``` diff --git a/pyclient.py b/pyclient.py new file mode 100644 index 0000000..ca1b5f6 --- /dev/null +++ b/pyclient.py @@ -0,0 +1,252 @@ +import argparse +import base64 +import json +import logging +import socket +import ssl +import sys +import urllib.parse +from typing import List, TypedDict, NotRequired, Union +from urllib.parse import ParseResult + +import certifi +import dns.resolver +import httptools + + +class HTTPResponseParser: + def __init__(self): + self.headers = {} + self.body = bytearray() + self.status_code = None + self.reason = None + self.http_version = None + self.parser = httptools.HttpResponseParser(self) + + def on_status(self, status): + self.reason = status.decode("utf-8", errors="replace") + + def on_header(self, name, value): + self.headers[name.decode("utf-8")] = value.decode("utf-8") + + def on_body(self, body): + self.body.extend(body) + + def feed_data(self, data): + self.parser.feed_data(data) + + +def parse_http_response(response_bytes): + parser = HTTPResponseParser() + parser.feed_data(response_bytes) + return { + "status_code": parser.parser.get_status_code(), + "reason": parser.reason, + "headers": parser.headers, + "body": bytes(parser.body), + } + + +def svcbname(parsed: ParseResult): + """Derive DNS name of SVCB/HTTPS record corresponding to target URL""" + if parsed.scheme == "https": + if (parsed.port or 443) == 443: + return parsed.hostname + else: + return f"_{parsed.port}._https.{parsed.hostname}" + elif parsed.scheme == "http": + if (parsed.port or 80) in (443, 80): + return parsed.hostname + else: + return f"_{parsed.port}._https.{parsed.hostname}" + else: + # For now, no other scheme is supported + return None + + +def get_ech_configs(domain) -> List[bytes]: + try: + answers = dns.resolver.resolve(domain, "HTTPS") + except dns.resolver.NoAnswer: + logging.warning(f"No HTTPS record found for {domain}") + return [] + except Exception as e: + logging.critical(f"DNS query failed: {e}") + sys.exit(1) + + configs = [] + + for rdata in answers: + if hasattr(rdata, "params"): + params = rdata.params + echconfig = params.get(5) + if echconfig: + configs.append(echconfig.ech) + + if len(configs) == 0: + logging.warning(f"No echconfig found in HTTPS record for {domain}") + + return configs + +def get_http(hostname, port, path, ech_configs) -> bytes: + logging.debug("Performing GET request for https://{hostname}:{port}/{path}") + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.load_verify_locations(certifi.where()) + context.options |= ssl.OP_ECH_GREASE + for config in ech_configs: + try: + context.set_ech_config(config) + context.check_hostname = False + except ssl.SSLError as e: + logging.error(f"SSL error: {e}") + pass + with socket.create_connection((hostname, port)) as sock: + with context.wrap_socket(sock, server_hostname=hostname, do_handshake_on_connect=False) as ssock: + try: + ssock.do_handshake() + logging.debug("Handshake completed with ECH status: %s", ssock.get_ech_status().name) + logging.debug("Inner SNI: %s, Outer SNI: %s", ssock.server_hostname, ssock.outer_server_hostname) + except ssl.SSLError as e: + retry_config = ssock._sslobj.get_ech_retry_config() + if retry_config: + logging.debug("Received a retry config: %s", base64.b64encode(retry_config)) + return get_http(hostname, port, path, [retry_config]) + logging.error(f"SSL error: {e}") + request = f'GET {path} HTTP/1.1\r\nHost: {hostname}\r\nConnection: close\r\n\r\n' + ssock.sendall(request.encode('utf-8')) + response = b'' + while True: + data = ssock.recv(4096) + if not data: + break + response += data + return response + + +def get(url): + parsed = urllib.parse.urlparse(url) + domain = parsed.hostname + ech_configs = get_ech_configs(svcbname(parsed)) + logging.debug("Discovered ECHConfig values: %s", [base64.b64encode(config) for config in ech_configs]) + request_path = (parsed.path or '/') + ('?' + parsed.query if parsed.query else '') + raw = get_http(domain, parsed.port or 443, request_path, ech_configs) + return parse_http_response(raw) + + +def cmd_get(url: str) -> None: + """Retrieves data from a given URL.""" + print(get(url)) + + +def cmd_echconfig(url: str) -> None: + """Print the bas64-encoded ECHConfig values for a given URL.""" + parsed = urllib.parse.urlparse(url) + for config in get_ech_configs(svcbname(parsed)): + print(base64.b64encode(config).decode("utf-8")) + + +class GetTarget(TypedDict): + description: NotRequired[str] + expected: NotRequired[str] + url: str + + +def read_targets_list() -> List[GetTarget]: + try: + input_json = sys.stdin.read() + input_data = json.loads(input_json) + + if not isinstance(input_data, list): + logging.critical("Invalid input format: JSON input must be a list") + sys.exit(1) + + for item in input_data: + if isinstance(item, dict): + if "url" not in item: + logging.error(f"Invalid input format, missing url: {item}") + sys.exit(1) + continue + if not isinstance(item, str): + logging.critical( + f"Invalid format: Each entry must be a string or object, but got {item}" + ) + sys.exit(1) + return input_data + except json.JSONDecodeError as e: + logging.critical(f"Error decoding JSON input: {e}") + sys.exit(1) + + +def cmd_getlist(demo: bool) -> None: + targets: List[Union[GetTarget, str]] + if demo: + targets = json.load(open("targets.json")) + else: + targets = read_targets_list() + for target in targets: + logging.debug("--------------------------------------------------------") + if isinstance(target, str): + cmd_get(target) + continue + logging.debug("Target description: %s", target["description"]) + logging.debug("Expected ECH status: %s", target["expected"]) + cmd_get(target["url"]) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="A Python HTTPS client with TLS ECH support.", + formatter_class=argparse.RawTextHelpFormatter, + ) + parser.add_argument( + "-v", "--verbose", action="store_true", help="Enable verbose logging" + ) + + subparsers = parser.add_subparsers( + title="subcommands", dest="command", help="Available subcommands" + ) + + echconfig_parser = subparsers.add_parser( + "echconfig", help="Print ECHConfig values from DNS (base64 encoded)." + ) + echconfig_parser.add_argument("url", help="URL to fetch config for.") + echconfig_parser.set_defaults(func=cmd_echconfig) + + get_parser = subparsers.add_parser("get", help="Fetch a URL.") + get_parser.add_argument("url", help="URL to fetch") + get_parser.set_defaults(func=cmd_get) + + getlist_parser = subparsers.add_parser( + "getlist", help="Iterate through a list of targets." + ) + getlist_parser.add_argument("--demo", help="Use a set of demo targets.", action="store_true") + getlist_parser.set_defaults(func=cmd_getlist) + + args = parser.parse_args() + + # Set up logging + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", + ) + + logging.debug(f"Command line arguments: {args}") + + if args.command is None: + parser.print_help() + return + + if args.command == "getlist": + args.func(args.demo) + return + + try: + args.func(args.url) + except AttributeError as e: + logging.critical( + f"Error: Subcommand '{args.command}' was called, but it requires no additional arguments: {e}" + ) + + +if __name__ == "__main__": + main() diff --git a/run_command.sh b/run_command.sh new file mode 100644 index 0000000..704a5da --- /dev/null +++ b/run_command.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -e + +export LD_LIBRARY_PATH=/usr/local/lib + +if [ "$#" -lt 1 ]; then + echo "Usage: $0 (curl|pyclient) [arguments...]" + exit 1 +fi + +command="$1" +shift # Remove the command from the argument list + +case "$command" in + shell) + bash "$@" + ;; + curl) + curl "$@" + ;; + pyclient) + /code/venv/bin/python pyclient.py "$@" + ;; + *) + echo "Unknown command: $command" + exit 1 + ;; +esac diff --git a/targets.json b/targets.json new file mode 100644 index 0000000..a9aaeac --- /dev/null +++ b/targets.json @@ -0,0 +1,122 @@ +[ + { + "description": "nginx server/minimal HTTPS RR", + "expected": "success", + "url": "https://min-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/nominal, HTTPS RR", + "expected": "success", + "url": "https://v1-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/nominal, HTTPS RR", + "expected": "success", + "url": "https://v2-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/two RRvals for nominal, minimal, HTTPS RR", + "expected": "success", + "url": "https://v3-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/three RRvals, 1st bad, 2nd good, 3rd bad, HTTPS RR", + "expected": "error", + "url": "https://v4-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/ECHConfigList with bad alg type (0xcccc) for ech kem", + "expected": "error", + "url": "https://bk1-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/zero-length ECHConfig within ECHConfigList", + "expected": "error", + "url": "https://bk2-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/ECHConfigList with bad ECH version (0xcccc)", + "expected": "error", + "url": "https://bv-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/nominal, HTTPS RR, bad alpn", + "expected": "client-dependent", + "url": "https://badalpn-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/20 values in HTTPS RR", + "expected": "success", + "url": "https://many-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/AliasMode (0) and ServiceMode (!=0) are not allowed together", + "expected": "error", + "url": "https://mixedmode-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/uses p256, hkdf-385 and chacha", + "expected": "success", + "url": "https://p256-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/two RRVALs one using x25519 and one with p256, same priority", + "expected": "success", + "url": "https://curves1-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/two RRVALs one using x25519 (priority=1) and one with p256 (priority=2)", + "expected": "success", + "url": "https://curves2-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/two RRVALs one using x25519 (priority=2) and one with p256 (priority=1)", + "expected": "success", + "url": "https://curves3-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/alpn is only h2", + "expected": "success", + "url": "https://h2alpn-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/alpn is only http/1.1", + "expected": "success", + "url": "https://h1alpn-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/alpn is http/1.1,foo,bar,bar,bom,h2", + "expected": "success", + "url": "https://mixedalpn-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/alpn is very long ending with http/1.1,h2", + "expected": "success", + "url": "https://longalpn-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/ECHConfiglist with 2 entries a 25519 one then a p256 one (both good keys)", + "expected": "success", + "url": "https://2thenp-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/ECHConfiglist with 2 entries a p256 one then a 25519 one (both good keys)", + "expected": "success", + "url": "https://pthen2-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server/minimal HTTPS RR but with 2 ECHConfig extensions", + "expected": "success", + "url": "https://withext-ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "nginx server", + "expected": "success", + "url": "https://ng.test.defo.ie/echstat.php?format=json" + }, + { + "description": "apache server", + "expected": "success", + "url": "https://ap.test.defo.ie/echstat.php?format=json" + } +]