From 3777d0831e665297f40191011a32fa1880f5052b Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Wed, 10 Aug 2022 15:53:39 -0700 Subject: [PATCH 01/23] Move 1.0 circle.yml to new 2.0 location https://circleci.com/blog/circleci-2-0-migration-best-practices/ --- circle.yml => .circleci/config.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename circle.yml => .circleci/config.yml (100%) diff --git a/circle.yml b/.circleci/config.yml similarity index 100% rename from circle.yml rename to .circleci/config.yml From 6a1b977f40704734f7bc0728eed81c07781bb623 Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Tue, 23 Aug 2022 14:41:14 -0700 Subject: [PATCH 02/23] Upgrade CircleCI Docker image to "next-gen" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On CircleCI, get the message: You’re using a deprecated Docker convenience image. Upgrade to a next-gen Docker convenience image. With a link to: https://discuss.circleci.com/t/legacy-convenience-image-deprecation/41034#browser-testing-4 This fix is based on that --- .circleci/config.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e9b4a8f06..73649582a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,7 +33,10 @@ # # then translated from 1.0 to 2.0 with: https://circleci.com/docs/2.0/config-translation/ -version: 2 +version: 2.1 +# browser-tools orb that provides installers for various browsers: https://circleci.com/developer/orbs/orb/circleci/browser-tools +orbs: + browser-tools: circleci/browser-tools@1.4.0 jobs: build: working_directory: ~/mathquill/mathquill @@ -44,18 +47,12 @@ jobs: environment: CIRCLE_ARTIFACTS: /tmp/circleci-artifacts CIRCLE_TEST_REPORTS: /tmp/circleci-test-results - # In CircleCI 1.0 we used a pre-configured image with a large number of languages and other packages. - # In CircleCI 2.0 you can now specify your own image, or use one of our pre-configured images. - # The following configuration line tells CircleCI to use the specified docker image as the runtime environment for you job. - # We have selected a pre-built image that mirrors the build environment we use on - # the 1.0 platform, but we recommend you choose an image more tailored to the needs - # of each job. For more information on choosing an image (or alternatively using a - # VM instead of a container) see https://circleci.com/docs/2.0/executor-types/ - # To see the list of pre-built images that CircleCI provides for most common languages see - # https://circleci.com/docs/2.0/circleci-images/ + # Docker image with browser testing tools: https://discuss.circleci.com/t/legacy-convenience-image-deprecation/41034#browser-testing-4 docker: - image: circleci/node:lts-browsers steps: + # install browsers: https://circleci.com/developer/orbs/orb/circleci/browser-tools + - browser-tools/install-browser-tools # Machine Setup # If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each # The following `checkout` command checks out your code to your working directory. In 1.0 we did this implicitly. In 2.0 you can choose where in the course of a job your code should be checked out. From 87cb95c51b0e99fb9d2a58ffebd95c778087a5f1 Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Tue, 23 Aug 2022 17:40:01 -0700 Subject: [PATCH 03/23] Small fixes to SauceLabs script Doesn't seem likely to fix the issue, which is that requests to the Sauce Labs /js-tests API endpoint are returning "status": "test error" --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 73649582a..374c33b38 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -87,8 +87,8 @@ jobs: if [ -x sc-*-linux/bin/sc ]; then echo Using cached sc-*-linux/bin/sc else - time wget https://saucelabs.com/downloads/sc-4.7.1-linux.tar.gz - time tar -xzf sc-4.7.1-linux.tar.gz + time wget https://saucelabs.com/downloads/sc-4.8.1-linux.tar.gz + time tar -xzf sc-4.8.1-linux.tar.gz fi time sc-*-linux/bin/sc --user $SAUCE_USERNAME --api-key $SAUCE_ACCESS_KEY \ @@ -97,8 +97,8 @@ jobs: echo 'Sauce Connect failed, try redownloading (https://git.io/vSxsJ)' rm -rf * - time wget https://saucelabs.com/downloads/sc-4.7.1-linux.tar.gz - time tar -xzf sc-4.7.1-linux.tar.gz + time wget https://saucelabs.com/downloads/sc-4.8.1-linux.tar.gz + time tar -xzf sc-4.8.1-linux.tar.gz time sc-*-linux/bin/sc --user $SAUCE_USERNAME --api-key $SAUCE_ACCESS_KEY \ --readyfile ~/sauce_is_ready @@ -154,7 +154,7 @@ jobs: - run: command: |- # Screenshots: capture in the background while running unit tests - mkdir -p $CIRCLE_TEST_REPORTS/mocha + mkdir -p $CIRCLE_TEST_REPORTS/mocha $CIRCLE_TEST_REPORTS/junit # CircleCI expects test results to be reported in an JUnit/xUnit-style XML file: # https://circleci.com/docs/test-metadata/#a-namemochajsamocha-for-nodejs From cbacd4125aae16fac0eef67028367a838c0e0e89 Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Thu, 1 Sep 2022 10:35:22 -0700 Subject: [PATCH 04/23] Actually upgrade CircleCI Docker image to next-gen Oops, the previous commit did the prep work for it but didn't actually change the Docker image https://github.com/mathquill/mathquill/commit/6a1b977f40704734f7bc0728eed81c07781bb623#r82824447 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 374c33b38..f29e8b798 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,7 +49,7 @@ jobs: CIRCLE_TEST_REPORTS: /tmp/circleci-test-results # Docker image with browser testing tools: https://discuss.circleci.com/t/legacy-convenience-image-deprecation/41034#browser-testing-4 docker: - - image: circleci/node:lts-browsers + - image: cimg/node:lts-browsers steps: # install browsers: https://circleci.com/developer/orbs/orb/circleci/browser-tools - browser-tools/install-browser-tools From fa22f00be0ae86ab1245909f9cf36f49fd7411fb Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Thu, 1 Sep 2022 10:57:54 -0700 Subject: [PATCH 05/23] [CI WIP] Run unit tests "in the background" I think CI won't wait for it to finish so this is similar to commenting it out --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f29e8b798..7c06e035f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -194,7 +194,8 @@ jobs: && touch ~/screenshots_are_ready || echo EXIT STATUS $? | tee /dev/stderr > ~/screenshots_are_ready: } background: true - - run: |- + - run: + command: |- # Unit tests in the browser echo '1. Launch tests' @@ -247,6 +248,7 @@ jobs: echo '3. Exit with non-zero status code if any unit tests failed' exit "$(jq '.["js tests"][0].result.failures' Date: Thu, 1 Sep 2022 11:39:51 -0700 Subject: [PATCH 06/23] [CI] Install ImageMagick for stitching together screenshot pieces --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7c06e035f..b60714ff2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -252,6 +252,9 @@ jobs: - run: |- # Stitch together screenshots and diff against master + echo '-1. Install ImageMagick' + sudo apt-get install -y imagemagick + echo '0. Wait for screenshots to be ready' while [ ! -e ~/screenshots_are_ready ]; do sleep 1; done test -z "$(<~/screenshots_are_ready)" || exit 1 From 8e5568e077fc13e7c40623c3b427a5c1b485899e Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Thu, 1 Sep 2022 12:39:01 -0700 Subject: [PATCH 07/23] [CI] Add timings to screenshot-stitching script --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b60714ff2..6b44a6e70 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -253,15 +253,15 @@ jobs: # Stitch together screenshots and diff against master echo '-1. Install ImageMagick' - sudo apt-get install -y imagemagick + time sudo apt-get install -y imagemagick echo '0. Wait for screenshots to be ready' - while [ ! -e ~/screenshots_are_ready ]; do sleep 1; done + time while [ ! -e ~/screenshots_are_ready ]; do sleep 1; done test -z "$(<~/screenshots_are_ready)" || exit 1 echo '1. Stitch together pieces' - for img in $(ls $CIRCLE_ARTIFACTS/imgs/pieces/); do - convert $(ls -1 $CIRCLE_ARTIFACTS/imgs/pieces/$img/*.png | sort -n) -append $CIRCLE_ARTIFACTS/imgs/$img.png + time for img in $(ls $CIRCLE_ARTIFACTS/imgs/pieces/); do + time convert $(ls -1 $CIRCLE_ARTIFACTS/imgs/pieces/$img/*.png | sort -n) -append $CIRCLE_ARTIFACTS/imgs/$img.png done echo '2. Download the latest screenshots from master' From 6bce2f9668706706c2354151437a1257ae908500 Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Thu, 1 Sep 2022 13:03:09 -0700 Subject: [PATCH 08/23] [CI] Custom script instead of Sauce Labs JS unit test API Sauce Labs JS unit test API appears to have stopped working sometime in the past month (in between this [1] build and this [2] re-run of the same build on the same commit), so write a script based on script/screenshots.js that manually calls Selenium WebDriver on SauceLabs to run the unit tests [1]: https://app.circleci.com/pipelines/github/mathquill/mathquill/72/workflows/8d798597-a6db-455f-8c20-16f86273e896/jobs/590 [2]: https://app.circleci.com/pipelines/github/mathquill/mathquill/72/workflows/4d48db1b-2363-4fe0-9f51-d14c655bd3e9/jobs/593 --- .circleci/config.yml | 47 +++-------------------------- script/unit_test_webdriver.js | 56 +++++++++++++++++++++++++++++++++++ test/unit.html | 37 +++++++---------------- 3 files changed, 71 insertions(+), 69 deletions(-) create mode 100644 script/unit_test_webdriver.js diff --git a/.circleci/config.yml b/.circleci/config.yml index 6b44a6e70..4a8e5d406 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -194,13 +194,9 @@ jobs: && touch ~/screenshots_are_ready || echo EXIT STATUS $? | tee /dev/stderr > ~/screenshots_are_ready: } background: true - - run: - command: |- + - run: |- # Unit tests in the browser - echo '1. Launch tests' - echo - # https://circleci.com/docs/environment-variables/ build_name="CircleCI build #$CIRCLE_BUILD_NUM" if [ $CIRCLE_PR_NUMBER ]; then @@ -210,45 +206,10 @@ jobs: build_name="$build_name: $CIRCLE_BRANCH" fi build_name="$build_name @ ${CIRCLE_SHA1:0:7}" + export MQ_CI_BUILD_NAME="$build_name" - # "build" and "customData" parameters from: - # https://wiki.saucelabs.com/display/DOCS/Test+Configuration+Options#TestConfigurationOptions-TestAnnotation - set -o pipefail - curl -i -X POST https://saucelabs.com/rest/v1/$SAUCE_USERNAME/js-tests \ - -u $SAUCE_USERNAME:$SAUCE_ACCESS_KEY \ - -H 'Content-Type: application/json' \ - -d '{ - "name": "Unit tests, Mocha", - "build": "'"$build_name"'", - "customData": {"build_url": "'"$CIRCLE_BUILD_URL"'"}, - "framework": "mocha", - "url": "http://localhost:8000/test/unit.html?post_xunit_to=http://localhost:9000", - "platforms": [["", "Chrome", ""]] - }' \ - | tee /dev/stderr | tail -1 > js-tests.json - - echo '2. Wait for tests to finish:' - echo - # > Make the request multiple times as the tests run until the response - # > contains `completed: true` to the get the final results. - # https://wiki.saucelabs.com/display/DOCS/JavaScript+Unit+Testing+Methods - while true # Bash has no do...while >:( - do - sleep 5 - curl -i -X POST https://saucelabs.com/rest/v1/$SAUCE_USERNAME/js-tests/status \ - -u $SAUCE_USERNAME:$SAUCE_ACCESS_KEY \ - -H 'Content-Type: application/json' \ - -d @js-tests.json \ - | tee /dev/stderr | tail -1 > status.json - - # deliberately do `... != false` rather than `... == true` - # because unexpected values should break rather than infinite loop - [ "$(jq .completed @@ -59,31 +64,11 @@

Unit Tests

teardown(function() { $('#mock').empty(); }); var runner = mocha.run(); - if (post_xunit_to) { - // the following is based on https://github.com/saucelabs-sample-scripts/JavaScript/blob/4946c5cf0ab7325dce5562881dba7c28e30989e5/reporting_mocha.js - var failedTests = []; - runner.on('fail', function(test, err) { - function flattenTitles(test) { - var titles = []; - while (test.parent.title) { - titles.push(test.parent.title); - test = test.parent; - } - return titles.reverse(); - } - - failedTests.push({name: test.title, result: false, message: err.message, stack: err.stack, titles: flattenTitles(test) }); + runner.on('end', function() { + setTimeout(function() { + if (xunitCallback) xunitCallback(xunit); }); - - runner.on('end', function() { - setTimeout(function() { - $.post(post_xunit_to, xunit).complete(function() { - window.mochaResults = runner.stats; - window.mochaResults.reports = failedTests; - }); - }); - }); - } + }); From 47d0b266f70391a051f1df7dde0441223e0dcce5 Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Thu, 1 Sep 2022 13:34:28 -0700 Subject: [PATCH 09/23] [CI] Get rid of trivial server for saving results XML That was only to get around the Sauce Labs unit testing APIs weird limitation of truncating test results. The new handrolled unit testing Selenium WebDriver script suffers no such limitation and can directly write the test results XML file. --- .circleci/config.yml | 50 ++++++++++++-------------------------------- 1 file changed, 13 insertions(+), 37 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4a8e5d406..8d1c65759 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -154,45 +154,21 @@ jobs: - run: command: |- # Screenshots: capture in the background while running unit tests - mkdir -p $CIRCLE_TEST_REPORTS/mocha $CIRCLE_TEST_REPORTS/junit - # CircleCI expects test results to be reported in an JUnit/xUnit-style XML file: - # https://circleci.com/docs/test-metadata/#a-namemochajsamocha-for-nodejs - # Our unit tests are in a browser, so they can't write to a file, and Sauce - # apparently truncates custom data in their test result reports, so instead we - # POST to this trivial Node server on localhost:9000 that writes the body of - # any POST request to $CIRCLE_TEST_REPORTS/junit/test-results.xml - node -e ' - require("http").createServer(function(req, res) { - res.setHeader("Access-Control-Allow-Origin", "*"); - req.pipe(process.stdout); - req.on("end", res.end.bind(res)); - }) - .listen(9000); - console.error("listening on http://0.0.0.0:9000/"); - ' 2>&1 >$CIRCLE_TEST_REPORTS/junit/test-results.xml | { - # ^ note: `2>&1` must precede `>$CIRCLE_TEST_REPORTS/...` because - # shell redirect is like assignment; if it came after, then both - # stdout and stderr would be written to `xunit.xml` and nothing - # would be piped into here - - head -1 # wait for "listening on ..." to be logged - - # https://circleci.com/docs/environment-variables/ - build_name="CircleCI build #$CIRCLE_BUILD_NUM" - if [ $CIRCLE_PR_NUMBER ]; then - build_name="$build_name: PR #$CIRCLE_PR_NUMBER" - [ "$CIRCLE_BRANCH" ] && build_name="$build_name ($CIRCLE_BRANCH)" - else - build_name="$build_name: $CIRCLE_BRANCH" - fi - build_name="$build_name @ ${CIRCLE_SHA1:0:7}" - export MQ_CI_BUILD_NAME="$build_name" + # https://circleci.com/docs/environment-variables/ + build_name="CircleCI build #$CIRCLE_BUILD_NUM" + if [ $CIRCLE_PR_NUMBER ]; then + build_name="$build_name: PR #$CIRCLE_PR_NUMBER" + [ "$CIRCLE_BRANCH" ] && build_name="$build_name ($CIRCLE_BRANCH)" + else + build_name="$build_name: $CIRCLE_BRANCH" + fi + build_name="$build_name @ ${CIRCLE_SHA1:0:7}" + export MQ_CI_BUILD_NAME="$build_name" - time { test -d node_modules/wd || npm install wd; } - time node script/screenshots.js http://localhost:8000/test/visual.html \ - && touch ~/screenshots_are_ready || echo EXIT STATUS $? | tee /dev/stderr > ~/screenshots_are_ready: - } + time { test -d node_modules/wd || npm install wd; } + time node script/screenshots.js http://localhost:8000/test/visual.html \ + && touch ~/screenshots_are_ready || echo EXIT STATUS $? | tee /dev/stderr > ~/screenshots_are_ready: background: true - run: |- # Unit tests in the browser From 98325ac2f58f008113d1e5f82fd9af86d9595eb4 Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Thu, 1 Sep 2022 13:36:29 -0700 Subject: [PATCH 10/23] [CI] Try to fix Circle test results parsing by un-nesting results file We appear[1] to be using store_test_results correctly, maybe CircleCI doesn't like that the test results XML file is nested in a directory? [1]: https://circleci.com/docs/collect-test-data#a-namemochajsamocha-for-nodejs --- script/unit_test_webdriver.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/unit_test_webdriver.js b/script/unit_test_webdriver.js index 0f98c7d84..40549dad7 100644 --- a/script/unit_test_webdriver.js +++ b/script/unit_test_webdriver.js @@ -37,8 +37,8 @@ browserDriver.init({ var lines = resultsXML.split('\n'); console.log('Got results XML (' + lines.length + ' lines):\n' + lines.slice(0, 10).join('\n') + '\n...'); - fs.writeFileSync(baseDir + '/junit/test-results.xml', resultsXML); - console.log('Wrote to ' + baseDir + '/junit/test-results.xml'); + fs.writeFileSync(baseDir + '/test-results.xml', resultsXML); + console.log('Wrote to ' + baseDir + '/test-results.xml'); // TODO: exit status based on whether tests passed }) From d6deba1b578ced07bd6abb14ae5c5d7049c95458 Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Thu, 1 Sep 2022 13:59:17 -0700 Subject: [PATCH 11/23] [CI] Unit test webdriver script exit code depends on test failure Unit test webdriver script now parses the Mocha test results XML to see if any tests failed and if so, exit with a nonzero exit code, so that CircleCI will report a CI failure --- script/unit_test_webdriver.js | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/script/unit_test_webdriver.js b/script/unit_test_webdriver.js index 40549dad7..2d94c1ddf 100644 --- a/script/unit_test_webdriver.js +++ b/script/unit_test_webdriver.js @@ -21,6 +21,8 @@ if (!baseDir) { process.exit(1); } +var success = true + var browserDriver = wd.promiseChainRemote('ondemand.saucelabs.com', 80, username, accessKey); browserDriver.init({ browserName: 'Chrome', @@ -40,12 +42,21 @@ browserDriver.init({ fs.writeFileSync(baseDir + '/test-results.xml', resultsXML); console.log('Wrote to ' + baseDir + '/test-results.xml'); - // TODO: exit status based on whether tests passed + const [_, failures, errors] = lines[0].match(/ failures="(\d+)" errors="(\d+)"/) + console.log(failures + ' failures, ' + errors + ' errors'); + success = (failures + errors === 0); }) .fail(function(err) { console.log('ERROR:', JSON.stringify(err, null, 2)); + success = false; +}) +.then(function() { + return browserDriver.sauceJobStatus(success); }) -.quit(); +.quit() +.then(function() { + process.exit(success ? 0 : 1); +}); function willLog() { var msg = [].join.call(arguments, ' '); From 0110f65ccd7e955c46f5787c969fc6f0147c5ee2 Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Thu, 1 Sep 2022 14:12:57 -0700 Subject: [PATCH 12/23] [CI] Fix result XML parsing --- script/unit_test_webdriver.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/unit_test_webdriver.js b/script/unit_test_webdriver.js index 2d94c1ddf..b571c6882 100644 --- a/script/unit_test_webdriver.js +++ b/script/unit_test_webdriver.js @@ -44,7 +44,7 @@ browserDriver.init({ const [_, failures, errors] = lines[0].match(/ failures="(\d+)" errors="(\d+)"/) console.log(failures + ' failures, ' + errors + ' errors'); - success = (failures + errors === 0); + success = (parseInt(failures) + parseInt(errors) === 0); }) .fail(function(err) { console.log('ERROR:', JSON.stringify(err, null, 2)); From 0e6b3377bd3daca0c3543f72ab3f8098c8c7009d Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Thu, 1 Sep 2022 14:25:24 -0700 Subject: [PATCH 13/23] [CI] Move get_xunit() function earlier Hopefully helps with this strange error (hopefully a race condition) where get_xunit() wasn't available when the unit test webdriver script tried to call it: https://app.circleci.com/pipelines/github/mathquill/mathquill/83/workflows/55ed0a40-c962-4b6a-98ad-714d55cd7b2d/jobs/604 --- test/unit.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/unit.html b/test/unit.html index 7ebae076b..ad0082f4a 100644 --- a/test/unit.html +++ b/test/unit.html @@ -10,6 +10,11 @@ window.onerror = function() { document.documentElement.style.background = 'red'; }; + + var xunitCallback; + function get_xunit(cb) { + xunitCallback = cb; + } @@ -33,11 +38,6 @@ var xunit = ''; Mocha.process.stdout.write = function(line) { xunit += line; }; - - var xunitCallback; - function get_xunit(cb) { - xunitCallback = cb; - } From bc79baab4198c88951ed66dddc5eb9d5d7ccc99b Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Thu, 1 Sep 2022 14:51:09 -0700 Subject: [PATCH 14/23] [CI] Fix Many-Worlds commit status API call Create and use a Personal Access Token with scope restricted to just GitHub commit statuses, instead of a bot account. I'm not sure what was meant by "API key", GitHub doesn't seem to offer those anymore, I don't remember how to access the MathQuillBot GitHub account, and the scope-restricted personal access token seems secure enough --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8d1c65759..843441ea3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -135,7 +135,7 @@ jobs: - run: |- # Generate link to Many-Worlds build and add to GitHub Commit Status curl -i -X POST https://api.github.com/repos/mathquill/mathquill/statuses/$CIRCLE_SHA1 \ - -u MathQuillBot:$GITHUB_STATUS_API_KEY \ + -u laughinghan:$GITHUB_STATUS_ACCESS_TOKEN \ -d '{ "context": "ci/many-worlds", "state": "success", From 6fae10a8af7c88695337c32695df3a98735944db Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Thu, 1 Sep 2022 15:01:45 -0700 Subject: [PATCH 15/23] [CI] Simplify unit test webdriver waiting for test completion And hopefully make it more robust to this race condition: https://app.circleci.com/pipelines/github/mathquill/mathquill/86/workflows/65d1fe97-58b7-4ce6-b70b-4270522a9b4b/jobs/610 --- script/unit_test_webdriver.js | 6 +++--- test/unit.html | 7 +------ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/script/unit_test_webdriver.js b/script/unit_test_webdriver.js index b571c6882..653e4db33 100644 --- a/script/unit_test_webdriver.js +++ b/script/unit_test_webdriver.js @@ -2,7 +2,7 @@ // 1. You've installed wd with `npm install wd'. // 2. You've set the environment variables $SAUCE_USERNAME and $SAUCE_ACCESS_KEY. // 3. If the environment variable $CIRCLE_TEST_REPORTS is not set images will be saved in /tmp -// 4. The argument is a URL of test/unit.html, which defines a get_xunit() async function that this script will call via Selenium WebDriver. +// 4. The argument is a URL of test/unit.html?xunit=true // // The intention of this script is that it will be ran from CircleCI // @@ -33,8 +33,8 @@ browserDriver.init({ }) .get(url) .then(willLog('get', url)) -.safeExecuteAsync('get_xunit(arguments[0])') -.then(willLog('get_xunit()')) +.safeExecuteAsync('window.xunitCallback = arguments[0];') +.then(willLog('waited for xunitCallback()')) .then(function(resultsXML) { var lines = resultsXML.split('\n'); console.log('Got results XML (' + lines.length + ' lines):\n' + lines.slice(0, 10).join('\n') + '\n...'); diff --git a/test/unit.html b/test/unit.html index ad0082f4a..35b63e49f 100644 --- a/test/unit.html +++ b/test/unit.html @@ -10,11 +10,6 @@ window.onerror = function() { document.documentElement.style.background = 'red'; }; - - var xunitCallback; - function get_xunit(cb) { - xunitCallback = cb; - } @@ -66,7 +61,7 @@

Unit Tests

runner.on('end', function() { setTimeout(function() { - if (xunitCallback) xunitCallback(xunit); + if (window.xunitCallback) xunitCallback(xunit); }); }); From ec41cea2aacb2aec1963977e6ccfc4b56e9bbaf8 Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Thu, 1 Sep 2022 15:27:26 -0700 Subject: [PATCH 16/23] [CI] Show error message in sauce_is_ready if present I thought sauce_is_ready would only be non-empty if Sauce Connect exited twice, but in this build sauce_is_ready is non-empty even though Sauce Connect worked fine --- .circleci/config.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 843441ea3..de7792894 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -149,7 +149,11 @@ jobs: command: PORT=8000 make server background: true # Wait for tunnel to be ready (`make server` is much faster, no need to wait for it) - - run: while [ ! -e ~/sauce_is_ready ]; do sleep 1; done; touch ~/sauce_was_ready; test -z "$(<~/sauce_is_ready)" + - run: |- + while [ ! -e ~/sauce_is_ready ]; do sleep 1; done + touch ~/sauce_was_ready + cat ~/sauce_is_ready + test -z "$(<~/sauce_is_ready)" # This is based on your 1.0 configuration file or project settings - run: command: |- From dadd208aeabec0a65d0bed6b9032c350b1aa6f41 Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Thu, 1 Sep 2022 15:48:49 -0700 Subject: [PATCH 17/23] [CI] Refactor screenshots/unit test webdriver YAML prep --- .circleci/config.yml | 41 ++++++++++++++--------------------------- 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index de7792894..296e95c81 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -108,10 +108,23 @@ jobs: | tee /dev/stderr > ~/sauce_is_ready exit 1 background: true + - run: |- + # Format build name $MQ_CI_BUILD_NAME + # https://circleci.com/docs/environment-variables/ + build_name="CircleCI build #$CIRCLE_BUILD_NUM" + if [ $CIRCLE_PR_NUMBER ]; then + build_name="$build_name: PR #$CIRCLE_PR_NUMBER" + [ "$CIRCLE_BRANCH" ] && build_name="$build_name ($CIRCLE_BRANCH)" + else + build_name="$build_name: $CIRCLE_BRANCH" + fi + build_name="$build_name @ ${CIRCLE_SHA1:0:7}" + export MQ_CI_BUILD_NAME="$build_name" # The following line was run implicitly in your 1.0 builds based on what CircleCI inferred about the structure of your project. In 2.0 you need to be explicit about which commands should be run. In some cases you can discard inferred commands if they are not relevant to your project. - run: if [ -z "${NODE_ENV:-}" ]; then export NODE_ENV=test; fi - run: export PATH="~/mathquill/mathquill/node_modules/.bin:$PATH" - run: npm install + - run: npm install wd # Save dependency cache - save_cache: key: v1-dep-{{ .Branch }}-{{ epoch }} @@ -158,38 +171,12 @@ jobs: - run: command: |- # Screenshots: capture in the background while running unit tests - - # https://circleci.com/docs/environment-variables/ - build_name="CircleCI build #$CIRCLE_BUILD_NUM" - if [ $CIRCLE_PR_NUMBER ]; then - build_name="$build_name: PR #$CIRCLE_PR_NUMBER" - [ "$CIRCLE_BRANCH" ] && build_name="$build_name ($CIRCLE_BRANCH)" - else - build_name="$build_name: $CIRCLE_BRANCH" - fi - build_name="$build_name @ ${CIRCLE_SHA1:0:7}" - export MQ_CI_BUILD_NAME="$build_name" - - time { test -d node_modules/wd || npm install wd; } time node script/screenshots.js http://localhost:8000/test/visual.html \ && touch ~/screenshots_are_ready || echo EXIT STATUS $? | tee /dev/stderr > ~/screenshots_are_ready: background: true - run: |- # Unit tests in the browser - - # https://circleci.com/docs/environment-variables/ - build_name="CircleCI build #$CIRCLE_BUILD_NUM" - if [ $CIRCLE_PR_NUMBER ]; then - build_name="$build_name: PR #$CIRCLE_PR_NUMBER" - [ "$CIRCLE_BRANCH" ] && build_name="$build_name ($CIRCLE_BRANCH)" - else - build_name="$build_name: $CIRCLE_BRANCH" - fi - build_name="$build_name @ ${CIRCLE_SHA1:0:7}" - export MQ_CI_BUILD_NAME="$build_name" - - time { test -d node_modules/wd || npm install wd; } - time node script/unit_test_webdriver.js http://localhost:8000/test/unit.html?xunit=true + node script/unit_test_webdriver.js http://localhost:8000/test/unit.html?xunit=true - run: |- # Stitch together screenshots and diff against master From de7f70e118a3b6b6217646734b8fd9bd854e7335 Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Thu, 1 Sep 2022 15:55:31 -0700 Subject: [PATCH 18/23] [CI] Disable script timeout in unit test webdriver Hopefully addresses: https://app.circleci.com/pipelines/github/mathquill/mathquill/89/workflows/1e21a584-e2a7-44f8-9dbf-e1f2563f791c/jobs/623 --- script/unit_test_webdriver.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/unit_test_webdriver.js b/script/unit_test_webdriver.js index 653e4db33..e99778771 100644 --- a/script/unit_test_webdriver.js +++ b/script/unit_test_webdriver.js @@ -33,6 +33,8 @@ browserDriver.init({ }) .get(url) .then(willLog('get', url)) +.setAsyncScriptTimeout(Infinity) +.then(willLog('setAsyncScriptTimeout(Infinity)')) .safeExecuteAsync('window.xunitCallback = arguments[0];') .then(willLog('waited for xunitCallback()')) .then(function(resultsXML) { From 57dabdefa3905736c1321b581bf74fdba566185a Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Thu, 1 Sep 2022 15:59:50 -0700 Subject: [PATCH 19/23] [CI] Fix disabling of unit test webdriver script timeout Surely the max value of a (signed) int32 will fit in whatever format they're storing --- script/unit_test_webdriver.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/unit_test_webdriver.js b/script/unit_test_webdriver.js index e99778771..48ac728e5 100644 --- a/script/unit_test_webdriver.js +++ b/script/unit_test_webdriver.js @@ -33,8 +33,8 @@ browserDriver.init({ }) .get(url) .then(willLog('get', url)) -.setAsyncScriptTimeout(Infinity) -.then(willLog('setAsyncScriptTimeout(Infinity)')) +.setAsyncScriptTimeout(2**31 - 1) +.then(willLog('setAsyncScriptTimeout(2**31 - 1)')) .safeExecuteAsync('window.xunitCallback = arguments[0];') .then(willLog('waited for xunitCallback()')) .then(function(resultsXML) { From ba6b23ddae5e1f85ca0f3351887da32a4ecdb610 Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Thu, 1 Sep 2022 16:20:17 -0700 Subject: [PATCH 20/23] [CI] Reset sauce_is_ready upon Sauce Connect failure Apparently Sauce Connect actually writes some JSON stuff there upon failure: https://app.circleci.com/pipelines/github/mathquill/mathquill/92/workflows/80042bf1-3bcb-483e-8a06-41f4ccc052e6/jobs/635 --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 296e95c81..0967a6443 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,7 +96,8 @@ jobs: test -e ~/sauce_was_ready && exit echo 'Sauce Connect failed, try redownloading (https://git.io/vSxsJ)' - rm -rf * + cat ~/sauce_is_ready + rm -rf * ~/sauce_is_ready time wget https://saucelabs.com/downloads/sc-4.8.1-linux.tar.gz time tar -xzf sc-4.8.1-linux.tar.gz From 89b85ce821fa883b635d867726eca1889adb9cd9 Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Thu, 1 Sep 2022 16:27:59 -0700 Subject: [PATCH 21/23] [CI] Try to cache ImageMagick install --- .circleci/config.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0967a6443..2fe681d40 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -126,6 +126,7 @@ jobs: - run: export PATH="~/mathquill/mathquill/node_modules/.bin:$PATH" - run: npm install - run: npm install wd + - run: sudo apt-get install -y imagemagick # Save dependency cache - save_cache: key: v1-dep-{{ .Branch }}-{{ epoch }} @@ -143,6 +144,8 @@ jobs: # These cache paths were specified in the 1.0 config - ~/sauce-connect - ./node_modules + # Based on: https://xyxyx.org/posts/2019-04-09-saving-the-debian-package-cache-on-circleci.html + - /var/cache/apt/archives # Test # This would typically be a build job when using workflows, possibly combined with build # This is based on your 1.0 configuration file or project settings @@ -180,10 +183,6 @@ jobs: node script/unit_test_webdriver.js http://localhost:8000/test/unit.html?xunit=true - run: |- # Stitch together screenshots and diff against master - - echo '-1. Install ImageMagick' - time sudo apt-get install -y imagemagick - echo '0. Wait for screenshots to be ready' time while [ ! -e ~/screenshots_are_ready ]; do sleep 1; done test -z "$(<~/screenshots_are_ready)" || exit 1 From 9f3a1981697a745b472adb1c78375ce7df5c8ba3 Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Thu, 1 Sep 2022 17:03:45 -0700 Subject: [PATCH 22/23] [CI] Handle Sauce Connect 4.8.1 writing info to sauce_is_ready ba6b23ddae5e1f85ca0f3351887da32a4ecdb610 is mistaken, it's not that Sauce Connect is writing to sauce_is_ready upon failure, it's that upon failure we're upgrading to 4.8.1 which no longer leaves the readyfile empty. This commit clears the cache (by updating the cache key [1]) and checks specifically for the error message we write to sauce_is_ready, instead of assuming Sauce Connect will leave it empty. [1]: https://circleci.com/docs/caching#clearing-cache --- .circleci/config.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2fe681d40..2e0dd1ab3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -67,11 +67,11 @@ jobs: - restore_cache: keys: # This branch if available - - v1-dep-{{ .Branch }}- + - v2-dep-{{ .Branch }}- # Default branch if not - - v1-dep-master- + - v2-dep-master- # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly - - v1-dep- + - v2-dep- # This is based on your 1.0 configuration file or project settings - run: command: |- @@ -96,8 +96,7 @@ jobs: test -e ~/sauce_was_ready && exit echo 'Sauce Connect failed, try redownloading (https://git.io/vSxsJ)' - cat ~/sauce_is_ready - rm -rf * ~/sauce_is_ready + rm -rf * time wget https://saucelabs.com/downloads/sc-4.8.1-linux.tar.gz time tar -xzf sc-4.8.1-linux.tar.gz @@ -129,7 +128,7 @@ jobs: - run: sudo apt-get install -y imagemagick # Save dependency cache - save_cache: - key: v1-dep-{{ .Branch }}-{{ epoch }} + key: v2-dep-{{ .Branch }}-{{ epoch }} paths: # This is a broad list of cache paths to include many possible development environments # You can probably delete some of these entries @@ -170,7 +169,9 @@ jobs: while [ ! -e ~/sauce_is_ready ]; do sleep 1; done touch ~/sauce_was_ready cat ~/sauce_is_ready - test -z "$(<~/sauce_is_ready)" + # this is how you check if a string starts with another string in Bash + # https://stackoverflow.com/a/2172367/362030 + [[ "$(<~/sauce_is_ready)" != ERROR:* ]] # This is based on your 1.0 configuration file or project settings - run: command: |- From 61eef2ae51f68cdfa360d65ff9996f2040fb7c84 Mon Sep 17 00:00:00 2001 From: Han Seoul-Oh Date: Thu, 1 Sep 2022 17:10:58 -0700 Subject: [PATCH 23/23] [CI] Properly bump CircleCI cache key --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2e0dd1ab3..d8f84ec85 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -67,11 +67,11 @@ jobs: - restore_cache: keys: # This branch if available - - v2-dep-{{ .Branch }}- + - v3-dep-{{ .Branch }}- # Default branch if not - - v2-dep-master- + - v3-dep-master- # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly - - v2-dep- + - v3-dep- # This is based on your 1.0 configuration file or project settings - run: command: |- @@ -128,7 +128,7 @@ jobs: - run: sudo apt-get install -y imagemagick # Save dependency cache - save_cache: - key: v2-dep-{{ .Branch }}-{{ epoch }} + key: v3-dep-{{ .Branch }}-{{ epoch }} paths: # This is a broad list of cache paths to include many possible development environments # You can probably delete some of these entries