Initial commit

.gitattributes

@@ -0,0 +1,8 @@
+* text eol=lf
+*.png binary
+*.jpg binary
+*.sqlite3 binary
+*.xlsx binary
+*.xls binary
+*.pdf binary
+*.docx binary

.github/workflows/template_setup.yaml

@@ -0,0 +1,79 @@
+name: Run cookiecutter on first push
+
+on: [push]
+
+permissions:
+  actions: write
+  contents: write
+
+jobs:
+  run-cookiecutter:
+    if: ${{ !endsWith(github.repository, '-auto-template')  }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+          fetch-depth: 0
+          ref: ${{ github.head_ref }}
+
+      - name: Install cookiecutter
+        run: pip3 install cookiecutter
+
+      - uses: actions/github-script@v4
+        id: fetch-repo-and-user-details
+        with:
+          script: |
+            const query = `query($owner:String!, $name:String!) {
+              repository(owner:$owner, name:$name) {
+                name
+                description
+                owner {
+                  login
+                  ... on User {
+                    name
+                  }
+                  ... on Organization {
+                    name
+                  }
+                }
+              }
+            }`;
+            const variables = {
+              owner: context.repo.owner,
+              name: context.repo.repo
+            }
+            const result = await github.graphql(query, variables)
+            console.log(result)
+            return result
+
+      - name: Rebuild contents using cookiecutter
+        env:
+          INFO: ${{ steps.fetch-repo-and-user-details.outputs.result }}
+          TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          export REPO_NAME=$(echo $INFO | jq -r '.repository.name')
+          git config --global user.name "Cookie Cutter"
+          git config --global user.email "<>"
+          # Run cookiecutter
+          pushd /tmp
+          cookiecutter $GITHUB_WORKSPACE --no-input \
+            project_name=$REPO_NAME \
+            repo_name=$REPO_NAME \
+            github_id=$GITHUB_REPOSITORY \
+            description="$(echo $INFO | jq -r .repository.description)"
+          # move into generated project and push to replace current template
+          cd /tmp/$REPO_NAME
+          git remote add origin https://$GITHUB_ACTOR:[email protected]/$GITHUB_REPOSITORY.git
+          git push --force --set-upstream origin main
+
+      - name: "enable github pages workflow option"
+        uses: actions/github-script@v6
+        continue-on-error: true
+        with:
+          script: |
+            github.request('POST /repos/{owner}/{repo}/pages', {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              build_type: 'workflow'
+            })

.github/workflows/template_test.yaml

@@ -0,0 +1,29 @@
+name: Run meta pytest suite on repo
+
+on:
+  pull_request:
+  push:
+
+concurrency: 
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-test:
+    if: ${{ endsWith(github.repository, '-auto-template')  }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.head_ref }}
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10' 
+
+      - name: Install cookiecutter
+        run: pip install pytest cookiecutter poetry
+
+      - name: run pytest
+        run: python -m pytest

.gitignore

@@ -0,0 +1 @@
+*.venv

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "{{ cookiecutter.repo_name }}"]
+	path = {{ cookiecutter.repo_name }}
+	url = https://github.com/mysociety/template_data_repo/

cookie-readme.md

@@ -0,0 +1,12 @@
+{# This file is the template for the resulting repo's readme file #}
+# {{ cookiecutter.project_name }}
+
+[![badge](https://mybinder.org/badge.svg)](https://mybinder.org/v2/gh/{{ cookiecutter.github_id }}/HEAD)
+
+{{ cookiecutter.description }}
+
+This repository is available online at https://github.com/{{ cookiecutter.github_id }}
+
+If Github Pages are enabled, the URL is: https://mysociety.github.io/{{ cookiecutter.github_id.split("/")[1] }}/
+
+Instructions on using the features of this notebook (data publishing, notebook rendering, Github Pages) are available in [https://github.com/mysociety/data_common/blob/main/data-repo-readme.md](Data Common readme file).

cookiecutter.json

@@ -0,0 +1,14 @@
+{
+    "repo_name": "snake_case_repo_name",
+    "hyphenated": "{{ '-'.join(cookiecutter['repo_name'].lower().split()).replace('_', '-') }}",
+    "underscored": "{{ cookiecutter.hyphenated.replace('-', '_') }}",
+    "project_name": "{{ cookiecutter.hyphenated.replace('-', ' ').title() }}",
+    "github_id": "mysociety/{{ cookiecutter.repo_name }}",
+    "description": "A short description of the project.",
+    "_copy_without_render": [".git",
+                             "notebooks/_render_config/default.yaml",
+                             "src/data_common/.github",
+                             "src/data_common",
+                             ".github",
+                             "docs/theme"]
+}

hooks/post_gen_project.py

@@ -0,0 +1,103 @@
+import shutil
+import os
+import subprocess
+from pathlib import Path
+
+template_dir = r"{{ cookiecutter._template }}"
+if any(x in template_dir for x in ["https:", "gh:"]):
+    repo_name = template_dir.split("/")[-1]
+    template_dir = Path.home() / ".cookiecutters" / repo_name
+else:
+    template_dir = Path(template_dir)
+
+template_repo = "https://github.com/mysociety/template_data_repo"
+template_branch = "main"
+
+helper_repo = "https://github.com/mysociety/data_common"
+helper_branch = "main"
+
+
+# this was all a submodule in the template, now it stands alone. Need to copy across the git info.
+# this is made conditional because in a templating test it won't be set up this way, but also that's fine. 
+if os.environ.get("UPDATE_TO_LATEST", "true").lower() == "true":
+    Path(".git").unlink()
+    real_git_folder = Path(template_dir) / ".git" / "modules" / ("{" + "{ cookiecutter.repo_name }" + "}")
+    shutil.copytree(real_git_folder, ".git")
+    git_config = Path(".git", "config")
+    notebook_git_config = Path(".git","modules", "src", "data_common", "config")
+
+    # remove reference to the work tree above
+    with open(git_config, "r") as f:
+        lines = f.readlines()
+    with open(git_config, "w") as f:
+        for line in lines:
+            if "cookiecutter.repo_name" not in line:
+                f.write(line)
+
+    # remove reference to the work tree above
+    with open(notebook_git_config, "r") as f:
+        lines = f.readlines()
+    with open(notebook_git_config, "w") as f:
+        for line in lines:
+            if "cookiecutter.repo_name" not in line:
+                f.write(line)
+            else:
+                f.write("	worktree = ../../../../src/data_common\n")
+
+    # adjust the git directory for the notebook helper
+    with open(Path("src","data_common",".git"), "w") as file:
+        file.write("gitdir: ../../.git/modules/src/data_common")
+
+#copy example env to env
+shutil.copyfile(Path(".env-example"),
+                Path(".env"))
+
+# when doing this on windows, sometimes clones bad line endings. 
+# This fixes the bash file docker uses. 
+# replacement strings
+WINDOWS_LINE_ENDING = b'\r\n'
+UNIX_LINE_ENDING = b'\n'
+
+# relative or absolute file path, e.g.:
+file_path = Path("src","data_common", "bin", "packages_setup.bash")
+
+with open(file_path, 'rb') as open_file:
+    content = open_file.read()
+
+content = content.replace(WINDOWS_LINE_ENDING, UNIX_LINE_ENDING)
+
+with open(file_path, 'wb') as open_file:
+    open_file.write(content)
+
+# Lock the upstream docker image source at point of departure from template
+
+data_common_tag = subprocess.check_output("git submodule status src/data_common", shell=True).strip()
+data_common_tag = data_common_tag.replace(b"+", b"")
+data_common_tag = data_common_tag[:7]
+
+data_common_tag = b"data_common:sha-" + data_common_tag
+
+for d in ["Dockerfile"]:
+
+    with open(d, 'rb') as open_file:
+        content = open_file.read()
+
+    content = content.replace(b"data_common:latest", data_common_tag)
+
+    with open(d, 'wb') as open_file:
+        open_file.write(content)
+
+# remove templates we haven't already copied into the higher level
+bad_workflows = [Path(".github", "workflows", "template_meta_test.yaml")]
+
+for w in bad_workflows:
+    w.unlink()
+
+if os.environ.get("UPDATE_TO_LATEST", "true").lower() == "true":
+
+    # remove, we don't want this project to have a default origin of the template library
+    os.system(f'git remote rm origin')
+
+    # package all up in a little box
+    os.system("git add --all")
+    os.system('git commit -m "Post-templating commit"')

hooks/pre_gen_project.py

@@ -0,0 +1,102 @@
+import shutil
+import os
+from pathlib import Path
+
+template_dir = r"{{ cookiecutter._template }}"
+if any(x in template_dir for x in ["https:", "gh:"]):
+    repo_name = template_dir.split("/")[-1]
+    template_dir = Path.home() / ".cookiecutters" / repo_name
+else:
+    template_dir = Path(template_dir)
+    if template_dir.is_absolute() is False:
+        raise ValueError("If specifying a specific directory, it needs to be an absolute path")
+
+def amend_file(filepath: Path, replace: dict):
+    """
+    amend a file to use cookiecutter basics
+    """
+    with open(filepath, "r") as f:
+        txt = f.read()
+    for key, value in replace.items():
+        txt = txt.replace(key, value)
+    with open(filepath, "w") as f:
+        f.write(txt)
+    filename = str(filepath)
+    for key, value in replace.items():
+        filename = filename.replace(key, value)
+    if str(filepath) != filename:
+        filepath.rename(filename)
+
+
+repo_dir = template_dir / ("{" + "{ cookiecutter.repo_name }" + "}")
+
+template_repo = "https://github.com/mysociety/template_data_repo"
+template_branch = "main"
+
+helper_repo = "https://github.com/mysociety/data_common"
+helper_branch = "main"
+
+# allow a env variable to override the template updating to latest version
+# this allows testing of the template
+if os.environ.get("UPDATE_TO_LATEST", "true").lower() == "true":
+    print("using UPDATE_TO_LATEST")
+    print("updating submodule")
+    os.system(f"cd {template_dir} && git submodule update --init --recursive")
+    # update to latest version
+    print("resetting main to latest commit")
+    os.system(f'cd "{repo_dir}" && git reset --hard')
+
+    print("resetting origin to the template wiki and pullonig down latest")
+    os.system(f'cd "{repo_dir}" && git remote rm origin')
+    os.system(
+        f'cd "{repo_dir}" && git remote add origin "{template_repo}" && git fetch origin && git pull origin main && git checkout main'
+    )
+    print("doing the same for the data_common repo")
+    os.system(f'cd "{repo_dir}" && cd src/data_common && git remote rm origin')
+    os.system(
+        f'cd "{repo_dir}" && cd src/data_common && git remote add origin "{helper_repo}" && git fetch origin && git pull origin main && git checkout main'
+    )
+    print("done")
+else:
+    print("UPDATE_TO_LATEST disabled.")
+
+source_readme = Path(template_dir, "cookie-readme.md")
+dest_readme = Path(repo_dir, "readme.md")
+general_readme = Path(repo_dir, "notebooks-readme.md")
+
+print(f"Copying {source_readme} to {dest_readme}")
+shutil.copyfile(source_readme, dest_readme)
+
+# Amend files that have a direct reference to the original name
+
+replace = {
+    "title: template_data_repo": "title: {" + "{ cookiecutter.project_name }" + "}",
+    'baseurl: "/template_data_repo"': 'baseurl: "/'
+    + "{"
+    + "{ cookiecutter.repo_name }"
+    + '}"',
+    "template_data_repo:${TAG:-latest}": "{" + "{ cookiecutter.repo_name }" + "}:${TAG:-latest}",
+    "template_data_repo": "{" + "{ cookiecutter.underscored }" + "}",
+    "Standardised template for mysociety data repositories": "{"
+    + "{ cookiecutter.description }"
+    + "}",
+}
+
+
+amend_file(Path(repo_dir, ".devcontainer", "devcontainer.json"), replace)
+amend_file(Path(repo_dir, "pyproject.toml"), replace)
+amend_file(Path(repo_dir, "docker-compose.yml"), replace)
+# amend_file(Path(repo_dir, "Dockerfile.dev"), replace)
+amend_file(Path(repo_dir, "Dockerfile"), replace)
+amend_file(Path(repo_dir, "tests", "test_template_data_repo.py"), replace)
+amend_file(Path(repo_dir, "docs", "index.md"), replace)
+amend_file(Path(repo_dir, "docs", "_config.yml"), replace)
+
+to_delete = [Path(repo_dir, ".github", "workflows", "docker-image.yml")]
+
+package_dir = Path(repo_dir, "src", "template_data_repo")
+package_dir.rename(Path(repo_dir, "src", "{" + "{ cookiecutter.underscored }" + "}"))
+
+for f in to_delete:
+    if f.exists():
+        f.unlink()

pytest.ini

@@ -0,0 +1,4 @@
+# pytest.ini
+[pytest]
+testpaths =
+    tests

readme.md

@@ -0,0 +1,24 @@
+# mySociety data repo template
+
+This is a self-formatting template for a [mySociety data repo](https://github.com/mysociety/template_data_repo). Based on [Simon Willison's approach](https://simonwillison.net/2021/Aug/28/dynamic-github-repository-templates/).
+
+Click the 'Use this template' button, and give a useful name (snakecase) and description, which will populate the readme.md of the new repo. 
+
+You can also use this link: https://github.com/mysociety/python-data-auto-template/generate
+
+If you have done this, and you are looking at this message in the new repo, wait twenty seconds and refresh. If this message is still here, investigate the status of the Github Action in the 'Actions' tab of the new repo.
+
+## Templating locally
+
+You can also use [Cookie Cutter](https://github.com/cookiecutter/cookiecutter).
+
+To create a new blank notebook template:
+
+```bash
+# if python installed but cookiecutter isn't
+pip install cookiecutter
+# then
+python -m cookiecutter gh:mysociety/python-data-auto-template
+```
+
+You will be prompted on setup settings.