incorporate review + smaller worker + prune

leap-stc · Aug 22, 2024 · 1c33507 · 1c33507
1 parent 5d32583
commit 1c33507
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 4 deletions.
diff --git a/.github/workflows/deploy_recipe.yaml b/.github/workflows/deploy_recipe.yaml
@@ -25,7 +25,7 @@ jobs:
       - name: "Install dependencies"
         run: |
           python -m pip install --upgrade pip
-          pip install git+https://github.com/leap-stc/pangeo-forge-runner
+          pip install pangeo-forge-runner
       - name: "Deploy recipes"
         run: |
           pangeo-forge-runner bake \

diff --git a/configs/config_dataflow.py b/configs/config_dataflow.py
@@ -4,12 +4,11 @@
 repo_path = os.environ['GITHUB_REPOSITORY']
 FEEDSTOCK_NAME = repo_path.split('/')[-1]
 
-c.Bake.prune = False
+c.Bake.prune = True
 c.Bake.bakery_class = "pangeo_forge_runner.bakery.dataflow.DataflowBakery"
 c.DataflowBakery.use_dataflow_prime = False
-c.DataflowBakery.machine_type = "e2-highmem-16"
+c.DataflowBakery.machine_type = "e2-highmem-8" # 1 year had max 50GB of ram on single worker. This is 64GB
 c.DataflowBakery.disk_size_gb = 400
-c.DataflowBakery.use_shuffle = False
 c.DataflowBakery.max_num_workers = 1
 c.DataflowBakery.use_public_ips = True
 c.DataflowBakery.service_account_email = (