generate: Put results in s3

This assumes an env has aws credentials to the instruct-lab-bot bucket. The last line of output is a URL to an index.html that lists all the files available associated with this generate run. Signed-off-by: Russell Bryant <[email protected]>
instructlab · Mar 14, 2024 · 47fba53 · 47fba53
1 parent 96e4f1c
commit 47fba53
Showing 1 changed file with 25 additions and 0 deletions.
diff --git a/scripts/generate.sh b/scripts/generate.sh
@@ -60,6 +60,31 @@ generate() {
     cd ..
     mkdir -p "$OUTPUT_DIR"
     lab generate --output-dir "$OUTPUT_DIR" --num-instructions "${NUM_INSTRUCTIONS}"
+    aws s3 cp --content-type text/plain --recursive "$OUTPUT_DIR" "s3://instruct-lab-bot/generate/${OUTPUT_DIR}"
+    cat << EOF > "$OUTPUT_DIR/index.html"
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Generated Data for ${OUTPUT_DIR}</title>
+</head>
+<body>
+    <h1>Generated Data for ${OUTPUT_DIR}</h1>
+    <ul>
+EOF
+    for file in "$OUTPUT_DIR"/*; do
+        if [ "$(basename ${file})" == "index.html" ]; then
+            continue
+        fi
+        URL=$(aws s3 presign --region us-east-2 "s3://instruct-lab-bot/generate/${file}")
+        echo "        <li><a href=\"${URL}\">$(basename ${file})</a></li>" >> "$OUTPUT_DIR/index.html"
+    done
+    cat << EOF >> "$OUTPUT_DIR/index.html"
+    </ul>
+</body>
+</html>
+EOF
+    aws s3 cp "$OUTPUT_DIR/index.html" "s3://instruct-lab-bot/generate/${OUTPUT_DIR}/index.html"
+    aws s3 presign --region us-east-2 "s3://instruct-lab-bot/generate/${OUTPUT_DIR}/index.html"
 }
 
 # Parse command line arguments