From ecfa3049a60e0202c45fd396d4689a80596b5e96 Mon Sep 17 00:00:00 2001 From: george haff Date: Wed, 11 Oct 2023 17:16:43 -0400 Subject: [PATCH 1/6] cruft removal attempt #1 --- .github/workflows/ci.yaml | 27 ------------------------- .github/workflows/performance-tests.yml | 2 -- TODO | 15 ++++++++++++++ deploy.json | 5 +---- dev/docker/python/setup.sh | 8 -------- dev/local/install.sh | 15 -------------- dev/local/setup.cfg | 25 ----------------------- 7 files changed, 16 insertions(+), 81 deletions(-) create mode 100644 TODO diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fd2d5fb4b..9b258f37a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,18 +5,6 @@ jobs: build: runs-on: ubuntu-latest steps: - - run: mkdir -p repos/undefx - - name: Checkout undefx/py3tester - uses: actions/checkout@v2 - with: - repository: undefx/py3tester - path: repos/undefx/py3tester - - name: Checkout undefx/undef-analysis - uses: actions/checkout@v2 - with: - repository: undefx/undef-analysis - path: repos/undefx/undef-analysis - - run: mkdir -p repos/delphi - name: Checkoutcmu-delphi/operations @@ -29,25 +17,10 @@ jobs: with: repository: cmu-delphi/utils path: repos/delphi/utils - - name: Checkout cmu-delphi/github-deploy-repo - uses: actions/checkout@v2 - with: - repository: cmu-delphi/github-deploy-repo - path: repos/delphi/github-deploy-repo - name: Checkout THIS REPO uses: actions/checkout@v2 with: path: repos/delphi/delphi-epidata - - name: Checkout cmu-delphi/flu-contest - uses: actions/checkout@v2 - with: - repository: cmu-delphi/flu-contest - path: repos/delphi/flu-contest - - name: Checkout cmu-delphi/nowcast - uses: actions/checkout@v2 - with: - repository: cmu-delphi/nowcast - path: repos/delphi/nowcast - name: Build docker images run: | diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index e5e5c4f1d..b459b2403 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -48,8 +48,6 @@ jobs: cd driver/repos/delphi git clone https://github.com/cmu-delphi/operations git clone https://github.com/cmu-delphi/utils - git clone https://github.com/cmu-delphi/flu-contest - git clone https://github.com/cmu-delphi/nowcast cd ../../ cd .. diff --git a/TODO b/TODO new file mode 100644 index 000000000..c90bea8ee --- /dev/null +++ b/TODO @@ -0,0 +1,15 @@ +- grep for: +undef +py3test +test_target +github.deploy.repo +flu.contest +nowcast (hard to filter on) + + +- still need to remove from: +docs/ +README.md +tests/ +integrations/ +??? diff --git a/deploy.json b/deploy.json index c1ee1ab07..378cc90af 100644 --- a/deploy.json +++ b/deploy.json @@ -241,10 +241,7 @@ "dst": "[[stasis]]/integrations/", "match": "^.*\\.(py)$", "recursive": true - }, - - "// run unit and coverage tests", - {"type": "py3test"} + } ] } diff --git a/dev/docker/python/setup.sh b/dev/docker/python/setup.sh index 80cc8d2a4..a01d40dcf 100644 --- a/dev/docker/python/setup.sh +++ b/dev/docker/python/setup.sh @@ -12,16 +12,8 @@ # bail if anything fails set -e -# create python package `undefx` -mkdir undefx -mv repos/undefx/py3tester/src undefx/py3tester -mv repos/undefx/undef-analysis undefx/undef_analysis - # create python package `delphi` mkdir delphi mv repos/delphi/operations/src delphi/operations mv repos/delphi/utils/src delphi/utils -mv repos/delphi/github-deploy-repo/src delphi/github_deploy_repo mv repos/delphi/delphi-epidata/src delphi/epidata -mv repos/delphi/flu-contest/src delphi/flu_contest -mv repos/delphi/nowcast/src delphi/nowcast diff --git a/dev/local/install.sh b/dev/local/install.sh index 9fd262e2e..c304b7686 100644 --- a/dev/local/install.sh +++ b/dev/local/install.sh @@ -17,12 +17,6 @@ # operations/ # delphi-epidata/ # utils/ -# flu-contest/ -# nowcast/ -# github-deploy-repo/ -# undefx/ -# py3tester/ -# undef-analysis/ # # Leaves you in driver, the main workdir. # @@ -33,15 +27,6 @@ cd driver/repos/delphi git clone https://github.com/cmu-delphi/operations git clone https://github.com/cmu-delphi/delphi-epidata git clone https://github.com/cmu-delphi/utils -git clone https://github.com/cmu-delphi/flu-contest -git clone https://github.com/cmu-delphi/nowcast -git clone https://github.com/cmu-delphi/github-deploy-repo -cd ../../ - -mkdir -p repos/undefx -cd repos/undefx -git clone https://github.com/undefx/py3tester -git clone https://github.com/undefx/undef-analysis cd ../../ ln -s repos/delphi/delphi-epidata/dev/local/Makefile diff --git a/dev/local/setup.cfg b/dev/local/setup.cfg index 1bb150963..62e591e5a 100644 --- a/dev/local/setup.cfg +++ b/dev/local/setup.cfg @@ -32,24 +32,6 @@ packages = delphi.epidata.server.endpoints delphi.epidata.server.endpoints.covidcast_utils delphi.epidata.server.utils - delphi.flu_contest - delphi.flu_contest - delphi.flu_contest.archefilter - delphi.flu_contest.covid - delphi.flu_contest.epicast - delphi.flu_contest.forecasters - delphi.flu_contest.hosp - delphi.flu_contest.main - delphi.flu_contest.uploads - delphi.flu_contest.utils - delphi.github_deploy_repo - delphi.github_deploy_repo.actions - delphi.nowcast - delphi.nowcast.experiments - delphi.nowcast.fusion - delphi.nowcast.obsolete - delphi.nowcast.sensors - delphi.nowcast.util delphi.operations delphi.operations.database_metrics delphi.operations.screenshots @@ -57,15 +39,8 @@ packages = delphi.utils delphi.utils.geo delphi.utils.obsolete - undefx.py3tester - undefx.undef_analysis package_dir = delphi.epidata = delphi/delphi-epidata/src - delphi.flu_contest = delphi/flu-contest/src - delphi.github_deploy_repo = delphi/github-deploy-repo/src - delphi.nowcast = delphi/nowcast/src delphi.operations = delphi/operations/src delphi.utils = delphi/utils/src - undefx.py3tester = undefx/py3tester/src - undefx.undef_analysis = undefx/undef-analysis From 1d1f36ae4e995b485bc26f028f32d96ef5913840 Mon Sep 17 00:00:00 2001 From: george haff Date: Wed, 11 Oct 2023 17:50:24 -0400 Subject: [PATCH 2/6] move setup script contents inside Dockerfile (or at least try to) --- dev/docker/python/Dockerfile | 15 +++++++++------ dev/docker/python/setup.sh | 19 ------------------- 2 files changed, 9 insertions(+), 25 deletions(-) delete mode 100644 dev/docker/python/setup.sh diff --git a/dev/docker/python/Dockerfile b/dev/docker/python/Dockerfile index 1836e4a5b..752c8a0ef 100644 --- a/dev/docker/python/Dockerfile +++ b/dev/docker/python/Dockerfile @@ -4,10 +4,13 @@ RUN apt-get update && apt-get install -y r-base && Rscript -e "install.packages( WORKDIR /usr/src/app -COPY repos repos -COPY repos/delphi/delphi-epidata/dev/docker/python/setup.sh . - RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime && \ - chmod -R o+r repos/ && \ - bash setup.sh && \ - pip install --no-cache-dir -r repos/delphi/delphi-epidata/requirements.api.txt -r repos/delphi/delphi-epidata/requirements.dev.txt + mkdir delphi && \ + chmod -R o+r delphi/ + +COPY repos/delphi/operations/src delphi/operations +COPY repos/delphi/utils/src delphi/utils +COPY repos/delphi/delphi-epidata/src delphi/epidata + +COPY repos/delphi/delphi-epidata/requirements.*.txt . +RUN install --no-cache-dir -r requirements.api.txt -r requirements.dev.txt diff --git a/dev/docker/python/setup.sh b/dev/docker/python/setup.sh deleted file mode 100644 index a01d40dcf..000000000 --- a/dev/docker/python/setup.sh +++ /dev/null @@ -1,19 +0,0 @@ -# This script sets up the correct directory structure within the `delphi_img` -# docker image. - -# Some notes on package structure: -# - Python package names can't contain hyphens, so hyphens in repo names are -# replaced with underscores in the package hierarchy. (An exception is the -# repo `delphi-epidata`, which is renamed to simply `epidata`.) -# - Repos are organized such that the main code for the package is inside of -# a `src/` directory. When deployed, `src/` is elided. (An exception is the -# legacy `undef-analysis` repo, which has sources at the top-level.) - -# bail if anything fails -set -e - -# create python package `delphi` -mkdir delphi -mv repos/delphi/operations/src delphi/operations -mv repos/delphi/utils/src delphi/utils -mv repos/delphi/delphi-epidata/src delphi/epidata From 6016c24f8dc74512906d9a8d7fb2b9c0df70a30d Mon Sep 17 00:00:00 2001 From: george haff Date: Wed, 11 Oct 2023 18:17:47 -0400 Subject: [PATCH 3/6] missed a pip\! --- dev/docker/python/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/docker/python/Dockerfile b/dev/docker/python/Dockerfile index 752c8a0ef..b3d1fb012 100644 --- a/dev/docker/python/Dockerfile +++ b/dev/docker/python/Dockerfile @@ -13,4 +13,4 @@ COPY repos/delphi/utils/src delphi/utils COPY repos/delphi/delphi-epidata/src delphi/epidata COPY repos/delphi/delphi-epidata/requirements.*.txt . -RUN install --no-cache-dir -r requirements.api.txt -r requirements.dev.txt +RUN pip install --no-cache-dir -r requirements.api.txt -r requirements.dev.txt From 10c30cd8a4543766003a8ab17e4b2a6f8b6fd4de Mon Sep 17 00:00:00 2001 From: george haff Date: Wed, 11 Oct 2023 18:40:37 -0400 Subject: [PATCH 4/6] stealing a trick from the dev Makefile, hoping it works --- .github/workflows/ci.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9b258f37a..be11f4429 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -53,11 +53,17 @@ jobs: - name: Run Unit Tests run: | - docker run --rm --network delphi-net --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/tests + docker run --rm --network delphi-net \ + --mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \ + --mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \ + --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/tests - name: Run Integration Tests run: | - docker run --rm --network delphi-net delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/integrations + docker run --rm --network delphi-net \ + --mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \ + --mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \ + delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/integrations - name: Clean Up run: | From bbdb34ce62e107844e4ef29a4be426fd9767234d Mon Sep 17 00:00:00 2001 From: george haff Date: Wed, 11 Oct 2023 18:50:50 -0400 Subject: [PATCH 5/6] syntax error fix? --- .github/workflows/ci.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index be11f4429..60316b245 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -54,16 +54,16 @@ jobs: - name: Run Unit Tests run: | docker run --rm --network delphi-net \ - --mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \ - --mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \ - --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/tests + --mount type=bind,source=./repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \ + --mount type=bind,source=./repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \ + --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/tests - name: Run Integration Tests run: | docker run --rm --network delphi-net \ - --mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \ - --mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \ - delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/integrations + --mount type=bind,source=./repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \ + --mount type=bind,source=./repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \ + delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/integrations - name: Clean Up run: | From 3ba08e404619a97966d3b9beb765f9cfa435efa0 Mon Sep 17 00:00:00 2001 From: george haff Date: Thu, 12 Oct 2023 12:20:04 -0400 Subject: [PATCH 6/6] documentation updates --- README.md | 25 +++++++++++-------------- TODO | 22 +++++++--------------- docs/epidata_development.md | 25 +++++++++++-------------- docs/new_endpoint_tutorial.md | 14 ++++---------- 4 files changed, 33 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 170c9a4ef..edb464d95 100644 --- a/README.md +++ b/README.md @@ -22,20 +22,17 @@ $ curl "https://raw.githubusercontent.com/cmu-delphi/delphi-epidata/dev/dev/loca You should now have the following directory structure: ```sh -├── driver -│ ├── .dockerignore -> repos/delphi/delphi-epidata/dev/local/.dockerignore -│ ├── Makefile -> repos/delphi/delphi-epidata/dev/local/Makefile -│ ├── repos -│ │ ├── pyproject.toml -> delphi/delphi-epidata/dev/local/pyproject.toml -│ │ ├── setup.cfg -> delphi/delphi-epidata/dev/local/setup.cfg -│ │ └── delphi -│ │ ├── delphi-epidata -│ │ ├── flu-contest -│ │ ├── github-deploy-repo -│ │ ├── nowcast -│ │ ├── operations -│ │ └── utils -``` +└── driver + ├── .dockerignore -> repos/delphi/delphi-epidata/dev/local/.dockerignore + ├── Makefile -> repos/delphi/delphi-epidata/dev/local/Makefile + └── repos + ├── pyproject.toml -> delphi/delphi-epidata/dev/local/pyproject.toml + ├── setup.cfg -> delphi/delphi-epidata/dev/local/setup.cfg + └── delphi + ├── delphi-epidata + ├── operations + └── utils + ``` and you should now be in the `driver` directory. You can now execute make commands diff --git a/TODO b/TODO index c90bea8ee..f17478d05 100644 --- a/TODO +++ b/TODO @@ -1,15 +1,7 @@ -- grep for: -undef -py3test -test_target -github.deploy.repo -flu.contest -nowcast (hard to filter on) - - -- still need to remove from: -docs/ -README.md -tests/ -integrations/ -??? +- for #1129 : + - grep for: + - py3test + - test_target + - and remove from: + - tests/* + - integrations/* diff --git a/docs/epidata_development.md b/docs/epidata_development.md index 29a0fdde7..0fe4005a6 100644 --- a/docs/epidata_development.md +++ b/docs/epidata_development.md @@ -19,17 +19,14 @@ $ curl "https://raw.githubusercontent.com/cmu-delphi/delphi-epidata/dev/dev/loca You should now have the following directory structure: ```sh -├── driver -│ ├── .dockerignore -> repos/delphi/delphi-epidata/dev/local/.dockerignore -│ ├── Makefile -> repos/delphi/delphi-epidata/dev/local/Makefile -│ ├── repos -│ │ └── delphi -│ │ ├── delphi-epidata -│ │ ├── flu-contest -│ │ ├── github-deploy-repo -│ │ ├── nowcast -│ │ ├── operations -│ │ └── utils +└── driver + ├── .dockerignore -> repos/delphi/delphi-epidata/dev/local/.dockerignore + ├── Makefile -> repos/delphi/delphi-epidata/dev/local/Makefile + └── repos + └── delphi + ├── delphi-epidata + ├── operations + └── utils ``` and you should now be in the `driver` directory. @@ -173,7 +170,7 @@ Then run the tests in a container based on that image: ```bash docker run --rm delphi_python \ - python3 -m undefx.py3tester.py3tester --color \ + python3 -m pytest \ repos/delphi/delphi-epidata/tests ``` @@ -315,7 +312,7 @@ More concretely, you can run Epidata API integration tests like this: ```bash docker run --rm --network delphi-net delphi_python \ - python3 -m undefx.py3tester.py3tester --color \ + python3 -m pytest \ repos/delphi/delphi-epidata/integrations ``` @@ -377,7 +374,7 @@ docker run --rm --network delphi-net \ --mount type=bind,source="$(pwd)"/repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \ --mount type=bind,source="$(pwd)"/repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \ delphi_python \ -python3 -m undefx.py3tester.py3tester --color \ +python3 -m pytest \ repos/delphi/delphi-epidata/integrations ``` diff --git a/docs/new_endpoint_tutorial.md b/docs/new_endpoint_tutorial.md index 6e6094161..be60b6814 100644 --- a/docs/new_endpoint_tutorial.md +++ b/docs/new_endpoint_tutorial.md @@ -33,16 +33,10 @@ tree -L 3 . ``` . └── repos -    ├── delphi -    │   ├── delphi-epidata -    │   ├── flu-contest -    │   ├── github-deploy-repo -    │   ├── nowcast -    │   ├── operations -    │   └── utils -    └── undefx -    ├── py3tester -    └── undef-analysis +    └── delphi +       ├── delphi-epidata +       ├── operations +       └── utils ``` # the data