From 8a81b412c166d4597b46c25619e8eae1e6164fc4 Mon Sep 17 00:00:00 2001 From: Patrice Chalin Date: Fri, 8 Nov 2024 05:14:44 -0500 Subject: [PATCH] [CI] Add GH workflow to run dev build using submodules at head (#5561) --- .github/workflows/build-dev.yml | 88 +++++++++++++++++++ package.json | 23 +++-- .../{sync-submodules.pl => pin-submodules.pl} | 13 +++ 3 files changed, 111 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/build-dev.yml rename scripts/{sync-submodules.pl => pin-submodules.pl} (65%) diff --git a/.github/workflows/build-dev.yml b/.github/workflows/build-dev.yml new file mode 100644 index 000000000000..fbf0c36e442c --- /dev/null +++ b/.github/workflows/build-dev.yml @@ -0,0 +1,88 @@ +name: Dev build + +on: + workflow_dispatch: + inputs: + submodule_path_regex: + description: + Regex of submodule paths to updated to HEAD before building. + default: content-modules + type: string + +jobs: + build-and-test: + name: BUILD and CHECK LINKS + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Create NPM cache-hash input file + run: | + mkdir -p tmp + jq '{devDependencies, dependencies, engines, gitHubActionCacheKey}' package.json > tmp/package-ci.json + + - name: Create and use reduced-dependencies package.json + run: | + jq '.devDependencies |= with_entries( + select(.key | test("prefix|hugo|css")) + ) + | del(.dependencies, .optionalDependencies)' \ + package.json > tmp/package-min.json + cp tmp/package-min.json package.json + + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + cache-dependency-path: tmp/package-ci.json + + - run: npm install --omit=optional + + - name: Fetch all submodules at HEAD + run: npm run update:submodule + + - name: + Revert to pinned versions of submodules except for + submodule_path_regex + run: npm run _sync -- ${{ inputs.submodule_path_regex }} + + - run: npm run log:test-and-fix + continue-on-error: true + env: + GET: no + + - uses: actions/upload-artifact@v4 + with: + name: build-log-etc + path: | + tmp/build-log.txt + tmp/package*.json + static/refcache.json + + check-refcache: + name: REFCACHE updates? + needs: build-and-test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: { name: build-log-etc } + - name: Fail when refcache contains entries with HTTP status 4XX + run: | + if grep -B 1 -e '"StatusCode": 4' static/refcache.json; then + echo "Run 'npx gulp prune' to remove 4xx entries from the refcache" + exit 1 + fi + - name: Does the refcache need updating? + run: npm run diff:fail + + check-build-log-for-issues: + name: WARNINGS in build log? + needs: build-and-test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: { name: build-log-etc } + - run: cat tmp/build-log.txt + - run: scripts/check-build-log.sh diff --git a/package.json b/package.json index c1aced6b3471..49dad6c0a0a1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,7 @@ { "Notes": [ - "The 'all' runs _all_ named scripts in sequence, even if one fails; and exits with failure in that case." + "The 'all' runs _all_ named scripts in sequence, even if one fails; and exits with failure in that case.", + "check:links actually has the side-effect of 'fixing' the refcache; this is why test-and-fix uses _fix:most" ], "scripts": { "__check:links": "make --keep-going check-links", @@ -20,6 +21,7 @@ "_diff:fail": "git diff --name-only --exit-code || exit 1", "_fail": "exit 1", "_filename-error": "echo 'ERROR: the following files violate naming conventions; fix using: `npm run fix:filenames`'; echo; npm run -s _ls-bad-filenames; exit 1", + "_fix:most": "npm run all -- $(npm -s run _list:fix:* | grep -Ev 'refcache|submodule')", "_get:no": "echo SKIPPING get operation", "_get:submodule:non-lang": "npm run _get:submodule -- content-modules/opentelemetry-specification themes/docsy", "_get:submodule": "set -x && git submodule update --init ${DEPTH:- --depth 999}", @@ -30,13 +32,13 @@ "_list:dict": "grep '^\\s*\"@cspell' package.json | awk -F: '{print $1}' | tr -d '\"'", "_list:fix:*": "npm run --loglevel=warn | grep -Ee '^\\s*fix:[^:]+$' | grep -v 'fix:all'", "_ls-bad-filenames": "find assets content static -name '*_*' ! -name '[_.]*'", + "_pin": "scripts/pin-submodules.pl", "_prebuild": "npm run seq -- get:submodule cp:spec", "_prepare:docsy": "cd themes/docsy && npm install", "_prettier:any": "npx prettier --ignore-path ''", "_rename-to-kebab-case": "find assets content static -name '*_*' ! -name '[_.]*' -exec sh -c 'mv \"$1\" \"${1//_/-}\"' _ {} \\;", "_serve:hugo": "hugo server --buildDrafts --minify", "_serve:netlify": "netlify dev -c \"npm run _serve:hugo -- --renderToMemory\"", - "_sync": "scripts/sync-submodules.pl", "all": "bash -c 'x=0; for c in \"$@\"; do npm run $c || x=$((x+1)); done; ((!x)) || (echo \"ERROR: some scripts failed!\" && exit 1)' -", "build:preview": "set -x && npm run _build -- --minify", "build:production": "npm run _hugo -- --minify", @@ -59,7 +61,7 @@ "cp:spec": "scripts/content-modules/cp-pages.sh", "diff:check": "npm run _diff:check || (echo; echo 'WARNING: the files above have not been committed'; echo)", "diff:fail": "npm run _diff:check || (echo; echo 'ERROR: the files above have changed. Locally rerun `npm run test-and-fix` and commit changes'; echo; exit 1)", - "fix:all": "npm run seq -- $(npm -s run _list:fix:*)", + "fix:all": "npm run all -- $(npm -s run _list:fix:*)", "fix:dict": "find content/en layouts -name \"*.md\" -print0 | xargs -0 scripts/normalize-cspell-front-matter.pl", "fix:filenames": "npm run _rename-to-kebab-case", "fix:format": "npm run format", @@ -69,12 +71,13 @@ "fix:i18n": "npm run fix:i18n:new", "fix:markdown": "npm run check:markdown -- --fix", "fix:refcache": "npm run check:links", - "fix:submodules": "npm run _sync", + "fix:submodules": "npm run update:submodule && npm run _pin", "fix:text": "npm run check:text -- --fix", "fix": "npm run fix:all", "format": "npm run _check:format -- --write && npm run _check:format:ja+zh -- --write", "get:submodule": "npm run _get:${GET:-submodule} --", "log:check:links": "npm run check:links | tee tmp/build-log.txt", + "log:test-and-fix": "npm run test-and-fix | tee tmp/build-log.txt", "make:public": "make public ls-public", "netlify-build:preview": "npm run seq -- build:preview diff:check", "netlify-build:production": "npm run seq -- build:production diff:check", @@ -93,17 +96,11 @@ "serve:hugo": "npm run _serve:hugo -- --renderToMemory", "serve:netlify": "npm run _serve:netlify", "serve": "npm run serve:hugo --", - "sync": "scripts/sync-submodules.pl", - "test-and-fix": "npm run seq -- check fix:dict fix:filenames", + "test-and-fix": "npm run seq -- fix:submodules _fix:most test", "test": "npm run check", - "update:docsy-dep": "npm install --save-dev autoprefixer@latest postcss-cli@latest", "update:htmltest-config": "scripts/htmltest-config.sh", - "update:hugo": "npm install --save-dev --save-exact hugo-extended@latest", - "update:hugo+": "npm run update:hugo && npm run update:docsy-dep", - "update:netlify": "npm install --save-optional netlify-cli@latest", - "update:other-pkg": "npm install --save-dev gulp@latest", + "update:pkg:hugo": "npm install --save-dev --save-exact hugo-extended@latest", "update:pkgs": "npx npm-check-updates -u", - "update:submodule:lang": "npm run seq -- update:submodule _get:submodule:non-lang", "update:submodule": "set -x && git submodule update --remote ${DEPTH:- --depth 999}" }, "devDependencies": { @@ -157,5 +154,5 @@ "proseWrap": "always", "singleQuote": true }, - "spelling": "cSpell:ignore docsy elemetry htmltest hugo loglevel netlify nowrap postnetlify prebuild precheck preinstall postbuild postget refcache textlint -" + "spelling": "cSpell:ignore docsy elemetry htmltest hugo loglevel netlify nowrap pkgs postnetlify prebuild precheck preinstall postbuild postget refcache textlint -" } diff --git a/scripts/sync-submodules.pl b/scripts/pin-submodules.pl similarity index 65% rename from scripts/sync-submodules.pl rename to scripts/pin-submodules.pl index 5bf3dc7b446e..3a928895e71e 100755 --- a/scripts/sync-submodules.pl +++ b/scripts/pin-submodules.pl @@ -1,9 +1,17 @@ #!/usr/bin/perl -w +# +# Switch submodules to their pinned versions, all submodules by default. Pass +# submodule-path regex to specify submodules to skip. use strict; use warnings; use FileHandle; +my $submodule_skip_regex = shift @ARGV; + +print "Using submodule-path skip regex: $submodule_skip_regex\n\n" + if $submodule_skip_regex; + my $file = '.gitmodules'; my $fh = FileHandle->new($file, 'r') or die "Error opening $file: $!"; my $content = do { local $/; <$fh> }; @@ -17,6 +25,11 @@ my $submodule_path = $submodules[$i]; my $commit = $submodules[$i + 1]; + if ($submodule_skip_regex && $submodule_path =~ /$submodule_skip_regex/) { + printf "Skipping $submodule_path\n"; + next; + } + my $command = "cd $submodule_path && git switch --detach $commit"; print "> $command\n"; system($command);