feat: initial version of gsm secret loader (#213)

This PR is the basis for the workload identity integration for GH
workflows.
Based on context it will load all secrets that are specified, transform
their names to screaming snake case and export them.

There are some tests and one is failing and i dont know why. All basic
usecases work though.

---------

Co-authored-by: Yannick Röder <[email protected]>

.github/workflows/_test-python-actions.yaml

@@ -0,0 +1,22 @@
+name: Test python-setup-poetry action
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - actions/parse-secret-definitions
+
+jobs:
+  tests:
+    name: Python action unit tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: bakdata/ci-templates/actions/[email protected]
+      - name: setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.12"
+      - name: Run all tests
+        run: |
+          find . -name "tests.py" -print0 | xargs -0i python -m unittest

.gitignore

@@ -8,3 +8,5 @@
 tmp*
 auto-doc*
 ./test*
+**/__pycache__/
+**/venv/

actions/gcp-gsm-load-secrets/README.md

@@ -0,0 +1 @@
+docs/actions/gcp-gsm-load-secrets/README.md

actions/gcp-gsm-load-secrets/action.yaml

@@ -0,0 +1,42 @@
+name: "Load secrets from Google Secret Manager"
+description: "Load secrets from Google Secret Manager and inject them into the environment"
+inputs:
+  gke-service-account:
+    description: "GKE service account for authentication"
+    required: true
+  gke-project-name:
+    description: "GKE project name for authentication"
+    required: true
+  workload-identity-provider:
+    description: "Workload identity provider for authentication"
+    required: true
+  secrets-to-inject:
+    description: "Secrets to inject into the environment"
+    required: true
+  export-to-environment:
+    description: "Export secrets to environment"
+    required: false
+    default: true
+outputs:
+  secrets:
+    description: "Secrets loaded from Secret Manager"
+    value: ${{ steps.secrets.outputs.secrets }}
+runs:
+  using: "composite"
+  steps:
+    - name: Authenticate at GCloud
+      uses: "google-github-actions/auth@v2"
+      with:
+        project_id: ${{ inputs.gke-project-name }}
+        workload_identity_provider: ${{ inputs.workload-identity-provider }}
+        service_account: ${{ inputs.gke-service-account }}
+    - id: "parse_secrets"
+      uses: "bakdata/ci-templates/actions/[email protected]"
+      with:
+        project_name: ${{ inputs.gke-project-name }}
+        secrets_list: ${{ inputs.secrets-to-inject }}
+    - id: "secrets"
+      uses: "google-github-actions/get-secretmanager-secrets@v2"
+      with:
+        secrets: ${{ steps.parse_secrets.outputs.secrets-list }}
+        export_to_environment: ${{ inputs.export-to-environment }}

actions/gcp-gsm-parse-secrets/Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3-slim AS builder
+RUN pip install poetry==1.8.2
+
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
+
+WORKDIR /app
+COPY pyproject.toml poetry.lock ./
+COPY main.py ./
+RUN poetry install --no-root && rm -rf $POETRY_CACHE_DIR
+
+# A distroless container image with Python and some basics like SSL certificates
+# https://github.com/GoogleContainerTools/dis/i/itroless
+FROM gcr.io/distroless/python3-debian12
+
+ENV VIRTUAL_ENV=/app/.venv \
+    PATH="/app/.venv/bin:$PATH"
+
+COPY --from=builder /app /app
+COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+
+WORKDIR /app
+ENV PYTHONPATH /app
+CMD ["/app/main.py"]

actions/gcp-gsm-parse-secrets/README.md

@@ -0,0 +1 @@
+docs/actions/gcp-gsm-parse-secrets/README.md

actions/gcp-gsm-parse-secrets/action.yaml

@@ -0,0 +1,15 @@
+name: "Parse secrets from GSM"
+description: "Transform secrets into a common format"
+inputs:
+  secrets-list:
+    description: "Secrets to inject into the environment"
+    required: true
+  project-name:
+    description: "GKE project name where the secrets are stored"
+    required: true
+outputs:
+  secrets-list:
+    description: "secret list with correct format"
+runs:
+  using: "docker"
+  image: "Dockerfile"

actions/gcp-gsm-parse-secrets/main.py

@@ -0,0 +1,66 @@
+import os
+import re
+
+import typer
+from typing_extensions import Annotated
+
+DEFAULT_DELIMITER = "!!!"
+
+# CAVEAT: will only work for one project at a time
+# to add secrets form another project, invoke the action a second time with the other project name
+
+
+# Set the output value by writing to the outputs in the Environment File, mimicking the behavior defined here:
+#  https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter
+def set_github_action_output(output_name, output_value, delim):
+    if os.environ.get("GITHUB_ACTION"):
+        f = open(os.path.abspath(os.environ["GITHUB_OUTPUT"]), "a")
+        f.write(
+            f"{output_name}<<{delim}\n{output_value}{delim}\n"
+        )  # ATTENTION: this might lead to problems if the output value contains the delimiter, which will not happen in this program but dont just copy this and expect it to work
+        f.close()
+    else:
+        print("would have set output", output_name, "to", output_value)
+
+
+# removes special characters and replace with underscores, successive special characters are replaced with a single underscore
+# convert to uppercase
+# if the secret would end in an underscore, remove it
+# format: SECRET_NAME:PROJECT_NAME/SECRET_NAME/VERSION
+def parse_secret(secret, project_name, delim=DEFAULT_DELIMITER):
+    if delim in secret:
+        raise ValueError(f"Invalid secret definition: {delim} is a reserved keyword")
+    components = secret.split("/")
+
+    if len(components) > 2:
+        raise ValueError(
+            f"Invalid secret definition: {secret}, not in the format 'secret_name/<version>'"
+        )
+    secret_name = re.sub("[^0-9a-zA-Z]+", "_", components[0]).upper().rstrip("_")
+    if secret_name == "":
+        raise ValueError(
+            f"Invalid secret definition: {components[0]} is not a valid secret name"
+        )
+    out = f"{secret_name}:{project_name}/{components[0]}"
+    if len(components) == 2 and len(components[1]) != 0:
+        out += f"/{components[1]}"
+    return out
+
+
+def main(
+    input_secrets: Annotated[str, typer.Argument(envvar="INPUT_SECRETS_LIST")],
+    gcp_project: Annotated[str, typer.Argument(envvar="INPUT_PROJECT_NAME")],
+    github_output_delimter: Annotated[str, typer.Argument()] = DEFAULT_DELIMITER,
+):
+    # Deduplicate the input secrets
+    input_secrets = set(input_secrets.splitlines())
+
+    output = ""
+    for secret in input_secrets:
+        output += parse_secret(secret, gcp_project, github_output_delimter) + "\n"
+
+    set_github_action_output("secrets-list", output, github_output_delimter)
+
+
+if __name__ == "__main__":
+    typer.run(main)

actions/gcp-gsm-parse-secrets/pyproject.toml

@@ -0,0 +1,15 @@
+[tool.poetry]
+name = "gcp-gsm-parse-secrets"
+version = "0.1.0"
+description = ""
+authors = ["Jan Max Tiedemann <[email protected]>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+typer = "^0.12.5"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

actions/gcp-gsm-parse-secrets/tests.py

@@ -0,0 +1,18 @@
+import unittest
+
+from main import parse_secret
+
+class TestParseSecret(unittest.TestCase):
+    def test_parse_secret(self):
+        self.assertEqual(parse_secret("secret_name", "project_name"), "SECRET_NAME:project_name/secret_name")
+        self.assertEqual(parse_secret("secret_name/version", "project_name"), "SECRET_NAME:project_name/secret_name/version")
+        self.assertEqual(parse_secret("123-456", "project_name"), "123_456:project_name/123-456")
+        self.assertEqual(parse_secret("123___123___123", "project_name"), "123_123_123:project_name/123___123___123")
+        self.assertEqual(parse_secret("i-like_trains__why_this?", "project_name"), "I_LIKE_TRAINS_WHY_THIS:project_name/i-like_trains__why_this?")
+
+    def test_parse_secret_special(self):
+        # FIXME: this test is failing and i dont know why
+        self.assertEqual(parse_secret("123&&123()123__123*__*_123", "project_name"), "123_123_123_123:project_name/123&&123()123__123*__*_123")
+
+if __name__ == '__main__':
+    unittest.main()