WIP - B2 based sync

RWKV · Feb 18, 2024 · 29c57b0 · 29c57b0
1 parent aa83aeb
commit 29c57b0
Show file tree

Hide file tree

Showing 8 changed files with 15 additions and 8 deletions.
diff --git a/.github/actions/gh-upload-output/action.yaml b/.github/actions/gh-upload-output/action.yaml
@@ -54,7 +54,7 @@ runs:
       if: always()
       run: |
         # Upload to backblaze
-        b2 upload-file $B2_BUCKET ./upload/ output-files/
+        b2 sync ./upload/ $B2_PATH_LM_EVAL_OUTPUT
 
     # This has been disabled, due to rate limits =[
     # ------------------------------------------------

diff --git a/.github/workflows/gh-task-runner-Alignment.yml b/.github/workflows/gh-task-runner-Alignment.yml
@@ -60,10 +60,11 @@ env:
   # Model HF repo
   MODEL_HF_REPO: ${{ github.event.inputs.model_hf_repo }}
 
-  # Secrets
+  # HF / B2 sync settings
   HUGGING_FACE_HUB_TOKEN: ${{secrets.HUGGING_FACE_HUB_TOKEN}}
   B2_APPLICATION_KEY_ID: ${{secrets.B2_APPLICATION_KEY_ID}}
   B2_APPLICATION_KEY: ${{secrets.B2_APPLICATION_KEY}}
+  B2_PATH_LM_EVAL_OUTPUT: ${{vars.B2_PATH_LM_EVAL_OUTPUT}}
 
 jobs:
   gh-task-runner:

diff --git a/.github/workflows/gh-task-runner-Error-Suite.yml b/.github/workflows/gh-task-runner-Error-Suite.yml
@@ -63,10 +63,11 @@ env:
   # Model HF repo
   MODEL_HF_REPO: ${{ github.event.inputs.model_hf_repo }}
 
-  # Secrets
+  # HF / B2 sync settings
   HUGGING_FACE_HUB_TOKEN: ${{secrets.HUGGING_FACE_HUB_TOKEN}}
   B2_APPLICATION_KEY_ID: ${{secrets.B2_APPLICATION_KEY_ID}}
   B2_APPLICATION_KEY: ${{secrets.B2_APPLICATION_KEY}}
+  B2_PATH_LM_EVAL_OUTPUT: ${{vars.B2_PATH_LM_EVAL_OUTPUT}}
 
 jobs:
   gh-task-runner:

diff --git a/.github/workflows/gh-task-runner-Few-Shot.yml b/.github/workflows/gh-task-runner-Few-Shot.yml
@@ -59,10 +59,11 @@ env:
   # Model HF repo
   MODEL_HF_REPO: ${{ github.event.inputs.model_hf_repo }}
 
-  # Secrets
+  # HF / B2 sync settings
   HUGGING_FACE_HUB_TOKEN: ${{secrets.HUGGING_FACE_HUB_TOKEN}}
   B2_APPLICATION_KEY_ID: ${{secrets.B2_APPLICATION_KEY_ID}}
   B2_APPLICATION_KEY: ${{secrets.B2_APPLICATION_KEY}}
+  B2_PATH_LM_EVAL_OUTPUT: ${{vars.B2_PATH_LM_EVAL_OUTPUT}}
 
 jobs:
   gh-task-runner:

diff --git a/.github/workflows/gh-task-runner-Large-Suite.yml b/.github/workflows/gh-task-runner-Large-Suite.yml
@@ -63,10 +63,11 @@ env:
   # Model HF repo
   MODEL_HF_REPO: ${{ github.event.inputs.model_hf_repo }}
 
-  # Secrets
+  # HF / B2 sync settings
   HUGGING_FACE_HUB_TOKEN: ${{secrets.HUGGING_FACE_HUB_TOKEN}}
   B2_APPLICATION_KEY_ID: ${{secrets.B2_APPLICATION_KEY_ID}}
   B2_APPLICATION_KEY: ${{secrets.B2_APPLICATION_KEY}}
+  B2_PATH_LM_EVAL_OUTPUT: ${{vars.B2_PATH_LM_EVAL_OUTPUT}}
 
 jobs:
   gh-task-runner:

diff --git a/.github/workflows/gh-task-runner-Single.yml b/.github/workflows/gh-task-runner-Single.yml
@@ -1722,10 +1722,11 @@ env:
   # Model HF repo
   MODEL_HF_REPO: ${{ github.event.inputs.model_hf_repo }}
 
-  # Secrets
+  # HF / B2 sync settings
   HUGGING_FACE_HUB_TOKEN: ${{secrets.HUGGING_FACE_HUB_TOKEN}}
   B2_APPLICATION_KEY_ID: ${{secrets.B2_APPLICATION_KEY_ID}}
   B2_APPLICATION_KEY: ${{secrets.B2_APPLICATION_KEY}}
+  B2_PATH_LM_EVAL_OUTPUT: ${{vars.B2_PATH_LM_EVAL_OUTPUT}}
 
 jobs:
   gh-task-runner:

diff --git a/.github/workflows/gh-task-runner-Slow-Suite.yml b/.github/workflows/gh-task-runner-Slow-Suite.yml
@@ -63,10 +63,11 @@ env:
   # Model HF repo
   MODEL_HF_REPO: ${{ github.event.inputs.model_hf_repo }}
 
-  # Secrets
+  # HF / B2 sync settings
   HUGGING_FACE_HUB_TOKEN: ${{secrets.HUGGING_FACE_HUB_TOKEN}}
   B2_APPLICATION_KEY_ID: ${{secrets.B2_APPLICATION_KEY_ID}}
   B2_APPLICATION_KEY: ${{secrets.B2_APPLICATION_KEY}}
+  B2_PATH_LM_EVAL_OUTPUT: ${{vars.B2_PATH_LM_EVAL_OUTPUT}}
 
 jobs:
   gh-task-runner:

diff --git a/.github/workflows/gh-task-runner-XLang.yml b/.github/workflows/gh-task-runner-XLang.yml
@@ -60,10 +60,11 @@ env:
   # Model HF repo
   MODEL_HF_REPO: ${{ github.event.inputs.model_hf_repo }}
 
-  # Secrets
+  # HF / B2 sync settings
   HUGGING_FACE_HUB_TOKEN: ${{secrets.HUGGING_FACE_HUB_TOKEN}}
   B2_APPLICATION_KEY_ID: ${{secrets.B2_APPLICATION_KEY_ID}}
   B2_APPLICATION_KEY: ${{secrets.B2_APPLICATION_KEY}}
+  B2_PATH_LM_EVAL_OUTPUT: ${{vars.B2_PATH_LM_EVAL_OUTPUT}}
 
 jobs:
   gh-task-runner: