From 7fdb8d8b9862e793fb37524f3e7ea3ea4613a7e2 Mon Sep 17 00:00:00 2001 From: Parker Lougheed Date: Wed, 17 Jan 2024 22:38:28 -0600 Subject: [PATCH 1/5] Introduce initial version of Dart-based site tooling --- .github/workflows/test.yml | 20 +- Dockerfile | 15 -- Makefile | 49 +---- README.md | 45 +--- dart_site.sh | 13 ++ pubspec.yaml | 4 +- tool/analyze-and-test-examples.sh | 199 ------------------ tool/check-code.sh | 18 -- tool/check-dart-sdk.sh | 91 -------- tool/check-formatting.sh | 68 ------ tool/check-links.sh | 41 ---- .../analysis_options.yaml | 0 tool/dart_site/bin/dart_site.dart | 30 +++ tool/dart_site/lib/dart_site.dart | 29 +++ .../lib/src/commands/analyze_dart.dart | 75 +++++++ .../dart_site/lib/src/commands/check_all.dart | 44 ++++ .../src/commands}/check_link_references.dart | 166 +++++++++------ .../lib/src/commands/check_links.dart | 124 +++++++++++ .../lib/src/commands/format_dart.dart | 68 ++++++ .../generate_effective_dart_toc.dart} | 142 +++++++++---- .../lib/src/commands/refresh_excerpts.dart | 182 ++++++++++++++++ .../dart_site/lib/src/commands/test_dart.dart | 87 ++++++++ .../src/commands/verify_firebase_json.dart | 152 +++++++++++++ tool/dart_site/lib/src/utils.dart | 88 ++++++++ .../pubspec.yaml | 12 +- tool/effective_dart_rules/.gitignore | 5 - tool/fetch-dart-sdk-sums.sh | 56 ----- tool/refresh-code-excerpts.sh | 107 ---------- tool/test.sh | 24 --- tool/update-dart-sums.sh | 34 --- tool/utils.sh | 24 --- 31 files changed, 1117 insertions(+), 895 deletions(-) create mode 100755 dart_site.sh delete mode 100755 tool/analyze-and-test-examples.sh delete mode 100755 tool/check-code.sh delete mode 100755 tool/check-dart-sdk.sh delete mode 100755 tool/check-formatting.sh delete mode 100755 tool/check-links.sh rename tool/{effective_dart_rules => dart_site}/analysis_options.yaml (100%) create mode 100644 tool/dart_site/bin/dart_site.dart create mode 100644 tool/dart_site/lib/dart_site.dart create mode 100644 tool/dart_site/lib/src/commands/analyze_dart.dart create mode 100644 tool/dart_site/lib/src/commands/check_all.dart rename tool/{dart_tools/bin => dart_site/lib/src/commands}/check_link_references.dart (61%) create mode 100644 tool/dart_site/lib/src/commands/check_links.dart create mode 100644 tool/dart_site/lib/src/commands/format_dart.dart rename tool/{effective_dart_rules/bin/main.dart => dart_site/lib/src/commands/generate_effective_dart_toc.dart} (50%) create mode 100644 tool/dart_site/lib/src/commands/refresh_excerpts.dart create mode 100644 tool/dart_site/lib/src/commands/test_dart.dart create mode 100644 tool/dart_site/lib/src/commands/verify_firebase_json.dart create mode 100644 tool/dart_site/lib/src/utils.dart rename tool/{effective_dart_rules => dart_site}/pubspec.yaml (53%) delete mode 100644 tool/effective_dart_rules/.gitignore delete mode 100755 tool/fetch-dart-sdk-sums.sh delete mode 100755 tool/refresh-code-excerpts.sh delete mode 100755 tool/test.sh delete mode 100755 tool/update-dart-sums.sh delete mode 100755 tool/utils.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 17f171e33f..05478c0992 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,23 +44,29 @@ jobs: with: sdk: ${{ matrix.sdk }} - run: dart pub get - - run: tool/test.sh - env: - DART_CHANNEL: ${{ matrix.sdk }} + - run: dart run dart_site format-dart --check + - run: dart run dart_site analyze-dart + - run: dart run dart_site test-dart + - run: dart run dart_site refresh-excerpts --fail-on-update linkcheck: name: Build site and check links runs-on: ubuntu-latest + strategy: + fail-fast: false steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 with: submodules: recursive - run: make build + - uses: dart-lang/setup-dart@ca7e6fee45ffbd82b555a7ebfc236d2c86439f5b + with: + sdk: stable + - run: dart pub get + - run: dart run dart_site check-link-references + - run: dart run dart_site verify-firebase-json - uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 with: node-version: ${{ env.NODE_VERSION }} - run: npm install -g firebase-tools@13.0.2 - - uses: dart-lang/setup-dart@ca7e6fee45ffbd82b555a7ebfc236d2c86439f5b - with: - sdk: stable - - run: tool/check-links.sh + - run: dart run dart_site check-links diff --git a/Dockerfile b/Dockerfile index c421c3cc81..bf8fb8b469 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,27 +57,12 @@ RUN set -eu; \ BASEURL="https://storage.googleapis.com/dart-archive/channels"; \ URL="$BASEURL/$DART_CHANNEL/release/$DART_VERSION/sdk/$SDK"; \ curl -fsSLO "$URL"; \ - echo "$DART_SHA256 *$SDK" | sha256sum --check --status --strict - || (\ - echo -e "\n\nDART CHECKSUM FAILED! Run 'make fetch-sums' for updated values.\n\n" && \ - rm "$SDK" && \ - exit 1 \ - ); \ unzip "$SDK" > /dev/null && mv dart-sdk "$DART_SDK" && rm "$SDK"; ENV PUB_CACHE="${HOME}/.pub-cache" RUN dart --disable-analytics RUN echo -e "Successfully installed Dart SDK:" && dart --version -# ============== DART-TESTS ============== -from dart as dart-tests -WORKDIR /app -COPY ./ ./ -RUN dart pub get -ENV BASE_DIR=/app -ENV TOOL_DIR=$BASE_DIR/tool -CMD ["./tool/test.sh"] - - # ============== NODEJS INSTALL ============== FROM dart as node diff --git a/Makefile b/Makefile index 55d7950742..f307a9548d 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,8 @@ -include .env -all: clean up down debug shell serve test-build test-run setup \ - serve emulate stage test build-image build deploy deploy-ci \ - fetch-sums test-builds test-run +all: clean up down shell serve test-build test-run run setup \ + serve emulate stage test build-image build deploy deploy-ci .PHONY: all .DEFAULT_GOAL := up @@ -67,15 +66,6 @@ serve: --incremental \ --trace -# Run all tests inside a built container -test: - DOCKER_BUILDKIT=1 docker build \ - -t dart-tests:${DART_CHANNEL} \ - --target dart-tests \ - --build-arg DART_VERSION=${DART_VERSION} \ - --build-arg DART_CHANNEL=${DART_CHANNEL} . - docker run --rm -v ${PWD}:/app dart-tests:${DART_CHANNEL} - # Build docker image with optional target # Usage: `make build-image [BUILD_CONFIGS=]` build-image: @@ -133,38 +123,3 @@ emulate: npx firebase emulators:start \ --only hosting \ --project ${FIREBASE_PROJECT} - - - -################## UTILS ################## - -# Fetch SDK sums for current Dart SDKs by arch -# This outputs a bash case format to be copied to Dockerfile -fetch-sums: - tool/fetch-dart-sdk-sums.sh \ - --version ${DART_VERSION} \ - --channel ${DART_CHANNEL} - -# Check Dart sums pulls the set of Dart SDK SHA256 hashes -# and writes them to a temp file. -check-sums: - tool/check-dart-sdk.sh - -# Update Dart sums replaces the Dart SDK SHA256 hashes -# in the Dockerfile and deletes the temp file. -update-sums: - tool/update-dart-sdk.sh - -# Test the dev container with pure docker -test-builds: - docker build -t ${BUILD_TAG}:stable \ - --no-cache --target=dart-tests . - docker build -t ${BUILD_TAG}:beta \ - --no-cache --target=dart-tests --build-arg DART_CHANNEL=beta . - docker build -t ${BUILD_TAG}:dev \ - --no-cache --target=dart-tests --build-arg DART_CHANNEL=dev . - -# Test stable run with volume -TEST_CHANNEL =? stable -test-run: - docker run --rm -it -v ${PWD}:/app ${BUILD_TAG}:${TEST_CHANNEL} bash diff --git a/README.md b/README.md index 1994a765a6..ae4e9fb99c 100644 --- a/README.md +++ b/README.md @@ -164,17 +164,14 @@ _choose one_ of the following submodule-cloning techniques: ### Checking documentation and example code If you've made changes to this site's documentation and/or example code, -and committed locally, then run the following command before pushing your work: +and committed locally, then run the following commands before pushing your work: ```terminal # Enter a running Docker container shell $ make run # Check/validate example code -$ tool/test.sh - -# Check links for 404 errors -$ tool/check-links.sh +$ dart_site check-all ``` If these scripts report errors or warnings, @@ -226,44 +223,6 @@ personal Firebase hosting staging site as follows: the staged version, the names of your reviewers, and so on. -## Creating and/or editing DartPad example code - -Most of the code used to create [DartPad][] examples is hosted on GitHub. -However, this repo also contains some `*.dart` files -responsible for DartPad example code. - -### DartPad picker - -The DartPad example picker must be manually compiled if changes are made. -This will regenerate the associated JavaScript file in `src/assets/dash/js`: - -```terminal -$ tool/compile.sh -``` - -## Dockerfile Maintenance - -### Dart SDK and Node PPA Checksum values - -Since the Dart SDK setup fetches remote files, -it's important to verify checksum values. -Both installs use `latest` and `lts` respectively, -so these files may be periodically updated. -When this happens, -local checksums may fail and **This will break the Docker/Compose setup/build**. -You will see the relevant output in your shell e.g. `DART CHECKSUM FAILED!...`. -When this happens, run the following command: - -```terminal -$ make fetch-sums -``` - -This command will output the updated checksum values for Dart, -and that output will be formatted similar -or the same as what is currently in the Dockerfile. -Copy this output and replace the relevant install code in the Dockerfile, -then rerun your setup/build again. - [Build Status SVG]: https://github.com/dart-lang/site-www/workflows/build/badge.svg [OpenSSF Scorecard SVG]: https://api.securityscorecards.dev/projects/github.com/dart-lang/site-www/badge [Scorecard Results]: https://deps.dev/project/github/dart-lang%2Fsite-www diff --git a/dart_site.sh b/dart_site.sh new file mode 100755 index 0000000000..23613ee493 --- /dev/null +++ b/dart_site.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +REPO_DIR=$(dirname $(dirname $BASH_SOURCE)) + +# Run dart pub get if the packages file doesn't exist yet. +if [[ ! -f "$REPO_DIR.dart_tool/package_config.json" ]]; then + dart pub get +fi + +# Run the dart_site tool and pass all arguments to it. +dart run dart_site "$@" diff --git a/pubspec.yaml b/pubspec.yaml index 3924cf31fc..03a65a2706 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,5 @@ name: site_www publish_to: none - homepage: https://dart.dev environment: @@ -12,5 +11,6 @@ dev_dependencies: path: site-shared/packages/code_excerpt_updater code_excerpter: path: site-shared/packages/code_excerpter - linkcheck: ^3.0.0 + dart_site: + path: tool/dart_site lints: ^3.0.0 diff --git a/tool/analyze-and-test-examples.sh b/tool/analyze-and-test-examples.sh deleted file mode 100755 index 8b27bb8e47..0000000000 --- a/tool/analyze-and-test-examples.sh +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/env bash -# Analyze dart files -# Final exit code reflects whether errors were encountered during tests -# Ignore script errors, so don't exit on errors/pipefail -set -u -source $TOOL_DIR/utils.sh - - -export PUB_ALLOW_PRERELEASE_SDK=quiet - -DART_VERSION=$(dart --version | perl -pe '($_)=/version: (\S+)/') -DART_CHANNEL=${DART_CHANNEL:-stable} -TMP=$BASE_DIR/tmp -EXAMPLES=$BASE_DIR/examples -PUB_ARG="upgrade" -LOG_FILE="$TMP/analyzer-output.txt" -EXIT_STATUS=0 -SAVE_LOGS=0 -QUICK=0 - -while [[ $# -gt 0 ]]; do - case "$1" in - --get) - PUB_ARG="get" - shift - ;; - --quick) - QUICK=1 - shift - ;; - --save-logs) - SAVE_LOGS=1 - shift - ;; - --tmp) - TMP=$2 - shift 2 - ;; - -h|--help) - echo "Usage: $(basename $0) [--get] [--quick] [--save-logs] [--help]" - exit 0 - ;; - *) - echo "Unsupported argument $1" >&2 - exit 1 - ;; - esac -done - -# Toggle analyzer comments in file via find/replace -# Usage: toggle_in_file_analyzer_flags [disable|reenable] [path] -function toggle_in_file_analyzer_flags() { - local action="$1" - local dir="$2" - local mark="!" - local toggle=" " - if [[ $action == 'disable' ]]; then - mark=" " - toggle="!" - fi - printf "\n$(blue "Toggling in-file flags: '$action'")\n" - find $dir -name "*.dart" ! -path "**/.*" -exec perl \ - -i -pe "s{//$mark(ignore(_for_file)?: .*?\b(stable|beta|dev)\b)}{//$toggle\$1}g" {} \; -} - -# Analyze and test code for arg $1 -# Usage: analyze_and_test /path/to/dir -function analyze_and_test() { - local project_dir="$1" - - printf "\n$(blue "--\nProcessing $project_dir...")\n" - pushd $project_dir > /dev/null - - dart pub $PUB_ARG - - EXPECTED_FILE="$project_dir/analyzer-results-$DART_CHANNEL.txt" - if [[ ! -e $EXPECTED_FILE ]]; then - EXPECTED_FILE="$project_dir/analyzer-results.txt" - fi - - toggle_in_file_analyzer_flags disable . - - dart analyze > $LOG_FILE || ( - echo -e "$(yellow "Ignoring analyzer exit code $?")" - ) - - if [[ -e $EXPECTED_FILE ]]; then - if grep -ve '^#' $EXPECTED_FILE | diff - $LOG_FILE > /dev/null; then - echo -e "$(blue "Analyzer output is as expected ($EXPECTED_FILE).")" - else - cat $LOG_FILE - echo -e "$(yellow "Unexpected analyzer output ($EXPECTED_FILE); here's the diff:")" - diff $LOG_FILE $EXPECTED_FILE || true - EXIT_STATUS=1 - if [[ -n $SAVE_LOGS ]]; then - cp $LOG_FILE $EXPECTED_FILE - fi - fi - elif grep -qvE '^Analyzing|^No issues found' $LOG_FILE; then - cat $LOG_FILE - - printf "\n$(yellow " - No analysis errors or warnings should be present in original source files. - Ensure that these issues are disabled using appropriate markers like: - // ignore_for_file: $DART_CHANNEL, some_analyzer_error_or_warning_id - Or if the errors are expected, create an analyzer-results.txt file. - ")\n" - - EXIT_STATUS=1 - if [[ -n $SAVE_LOGS ]]; then - cp $LOG_FILE $EXPECTED_FILE - fi - else - cat $LOG_FILE - fi - - toggle_in_file_analyzer_flags reenable . - - if [[ ! -d "test" ]]; then - echo -e "$(blue "Nothing to test in this project.")" - return - fi - - printf "\n$(blue "Running VM tests (exclude browser) ...")\n" - dart run test --exclude-tags=browser \ - | tee $LOG_FILE | $FILTER1 | $FILTER2 "$FILTER_ARG" - - PASSED=$(grep -E 'All tests passed!|^No tests ran' $LOG_FILE) - if [[ -z "$PASSED" ]]; then - printf "\n$(red "Found VM test errors")\n" - EXIT_STATUS=1 - fi - - # TODO(chalin): as of 2019/11/17, we don't need to select individual browser - # test files. Run browser tests over all files, since VM-only tests have - # been annotated as such. - BROWSER_TESTS=$(find . -name "*browser_test.dart" -o -name "*html_test.dart") - - if [[ -z $BROWSER_TESTS ]]; then - printf "\n$(blue "No browser-only tests - skipping")\n" - else - printf "\n$(blue "Running browser-only tests...")\n" - dart run test \ - --tags browser \ - --platform chrome $BROWSER_TESTS 2>&1 | \ - tee $LOG_FILE | $FILTER1 | $FILTER2 "$FILTER_ARG" - - PASSED=$(grep 'All tests passed!' $LOG_FILE) - if [[ -z "$PASSED" ]]; then - printf "\n$(red "Found browser-only test errors")\n\n" - EXIT_STATUS=1 - fi - fi - - popd > /dev/null -} - - -if [[ ! -e $TMP ]]; then - mkdir $TMP -fi - -# If quick, fail quick -if [[ -n "$QUICK" ]]; then - FILTER1="tr '\r' '\n'" - FILTER2="grep -E" - FILTER_ARG="(Some|All|No) tests" -else - FILTER1="cat -" - FILTER2="cat" - FILTER_ARG="-" -fi - - -printf "\n$(blue "Begin test and analyze... -TMP: $TMP -EXAMPLES: $EXAMPLES -DART_VERSION: $DART_VERSION -DART_CHANNEL: $DART_CHANNEL -")\n" - - -for dir in $EXAMPLES/??*; do - if [[ ! -d $dir || $dir == *util ]]; then - continue - fi - analyze_and_test $dir -done - -if [[ $EXIT_STATUS == 0 ]]; then - printf "\n$(blue "All tests passed for all suites!")\n\n" -else - printf "\n$(red " - Some packages have test failures and/or analysis errors. - Look at the full output from this script for details. - ")\n" -fi - -exit $EXIT_STATUS diff --git a/tool/check-code.sh b/tool/check-code.sh deleted file mode 100755 index a0eaccbca9..0000000000 --- a/tool/check-code.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -# Local development tool for checking formatting issues -# then refreshing code, with cache. - -set -eu -o pipefail -source $TOOL_DIR/utils.sh - -# Validate formatting in files. This will exit if there are -# any formatting fixes required in the examples directory. -$TOOL_DIR/check-formatting.sh - -printf "\n$(blue "Refreshing code excerpts...")" -( set -x - $TOOL_DIR/refresh-code-excerpts.sh --keep-dart-tool -) || ( - printf "\n$(red "+ Some code excerpts need to be refreshed")\n" - exit 1 -) diff --git a/tool/check-dart-sdk.sh b/tool/check-dart-sdk.sh deleted file mode 100755 index 6349fd9488..0000000000 --- a/tool/check-dart-sdk.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash -# Use this file locally to update Dart SDK checksum values in the Dockerfile -# Prints output similar to cases in Dockerfile for easy composition -# when having to update checksum values for updates dart SDK. -set -eu -o pipefail -TOOL_DIR="${TOOL_DIR:=$(dirname "$0")}" -source $TOOL_DIR/utils.sh - -VERSION="latest" -CHANNEL="stable" - -while (( "$#" )); do - case "$1" in - --version) - VERSION=$2 - shift 2 - ;; - --channel) - CHANNEL=$2 - shift 2 - ;; - *) - echo "Unsupported argument $1" >&2 - exit 1 - ;; - esac -done - -echo -e "\nPulling latest Dart SHA hashes.\n\nThis will take a moment.\n" - -BASEURL="https://storage.googleapis.com/dart-archive/channels" -CHANNELS="stable beta dev" -ARCHS="amd64 arm64" -ENDING='\\\n' -FILE=$TOOL_DIR/new-dart-hashes.txt - -true > $FILE - -for CHANNEL in $CHANNELS; do - for ARCH in $ARCHS; do - printf " ${ARCH}_${CHANNEL}) $ENDING" >> $FILE - _arch=$ARCH - if [[ "$_arch" == "amd64" ]]; then - _arch='x64' - fi - _filename="dartsdk-linux-${_arch}-release.zip" - _url="$BASEURL/$CHANNEL/release/$VERSION/sdk/$_filename" - curl -fsSLO $_url - _checksum=$(shasum -a 256 $_filename) - read -a _fname_arr <<< "${_checksum}" # Read in string output as array - _checkonly="${_fname_arr%:*}" # Remove filename portion of checksum output - printf " DART_SHA256=\"$_fname_arr\"; $ENDING" >> $FILE - printf " SDK_ARCH=\"$_arch\";; $ENDING" >> $FILE - echo "Pulled ${ARCH}_${CHANNEL}: $_fname_arr" - rm $_filename - done -done - -echo -e "\n\nPulled latest Dart SHA hashes and saved to $FILE.\n" - -lead='# BEGIN dart-sha$' -tail='# END dart-sha$' -new_file='tool/new-dart-hashes.txt' -existing_file='Dockerfile' - -new_hash=$(sed -n -e '/DART_SHA/ p' -e '/DART_SHA/ q' $new_file) -old_hash=$(sed -n -e '/DART_SHA/ p' -e '/DART_SHA/ q' $existing_file) - -echo -e "Old $old_hash" -echo -e "New $new_hash" - -# Compare the SHA hashes. -if [[ "$new_hash" == "$old_hash" ]]; then - echo -e "Current SHA hashes are the latest hashes.\n" - echo -e "No update needed.\n" - rm $new_file - echo -e "Removed $new_file.\n" - echo -e "Re-run check-dart-sdk.sh to pull the current SHA hashes.\n" -else - if [[ -f "$new_file" ]]; then - echo -e "Retrieved replacement hashes and saved to $new_file.\n" - if [[ -f "$existing_file" ]]; then - echo -e "Found Dockerfile at $existing_file.\n" - echo -e "Run tool/update-dart-sums.sh." - else - echo -e "No Dockerfile found." - fi - else - echo -e "No replacement hashes found at this time.\n" - fi -fi diff --git a/tool/check-formatting.sh b/tool/check-formatting.sh deleted file mode 100755 index f8804d4ad4..0000000000 --- a/tool/check-formatting.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# Point out whether submitted files are correctly formatted - -set -eu -o pipefail -source $TOOL_DIR/utils.sh - - -EXAMPLES="$BASE_DIR/examples" -FIX=0 - -while [[ $# -gt 0 ]]; do - case "$1" in - --fix) - FIX=1 - shift - ;; - -h|--help) - echo "Usage: $(basename $0) [--fix]" - exit 0 - ;; - *) - echo "Unsupported argument $1" >&2 - exit 1 - ;; - esac -done - -# Check formatting for all *.dart files in the examples -# directory and report back any files that need to be fixed. -function check_formatting() { - IFS=" " - read -a files <<< "$@" - local file_count=${#files[@]} - printf "\n$(blue "Checking formatting on $file_count files...")\n" - - local output="none" - if [[ $FIX -eq 1 ]]; then - output="write" - fi - - IFS=$'\n' - local results=($(dart format --output=$output "$@")) - unset results[-1] # Remove last line (summary) of format output - local error_count=${#results[@]} - if [[ $error_count -gt 0 ]]; then - printf "$(red "Found $error_count files that require(d) fixing:")\n\n" - IFS=' ' - for line in "${results[@]}"; do - read -r _ filepath <<< "$line" - printf " $(yellow $filepath)\n" - done - if [[ $FIX -eq 1 ]]; then - printf "\n$(red "These files have been fixed/written, please verify")\n\n"; - else - printf "\n$(red "Please fix the above files and commit your changes")\n\n"; - fi - exit 1; - else - printf "$(blue "0 files required formatting")\n\n" - fi -} - -dart_files=$( - find $EXAMPLES -name "*.dart" \ - ! -path "**/.*" \ - ! -path "**/build/**" -) -check_formatting $dart_files diff --git a/tool/check-links.sh b/tool/check-links.sh deleted file mode 100755 index e5e7a8ba2b..0000000000 --- a/tool/check-links.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# Check for non-200 links in built Jekyll site using Firebase emulator -set -eu -o pipefail -source $TOOL_DIR/utils.sh - -dart pub get - -echo "Checking for valid link references..." -# Check for invalid link references before checking for links -dart run tool/dart_tools/bin/check_link_references.dart -echo $'No invalid link references found!\n' - -trap clean_up SIGINT SIGTERM ERR EXIT - -EMULATOR_PORT=5500 # airplay runs on :5000 - -# Catch error, stop running emulator process by port -clean_up() { - trap - SIGINT SIGTERM ERR EXIT - echo -e "$(blue "Shutting down emulator...")" - lsof -t -i:$EMULATOR_PORT | xargs kill -9 > /dev/null 2>&1 - echo -e "$(blue "Done!")\n" -} - -echo -e "$(blue "Starting Firebase emulator async...")" -npx firebase emulators:start \ - --only hosting \ - --project default > /dev/null 2>&1 & -emulator_status=$? - -sleep 3 - -if [[ -z "$emulator_status" ]]; then - echo -e "$(red "Emulator did not start...")" - exit 1 -else - echo -e "$(blue "Emulator is running")" -fi - -SKIP_FILE="$TOOL_DIR/config/linkcheck-skip-list.txt" -dart run linkcheck :$EMULATOR_PORT --skip-file $SKIP_FILE diff --git a/tool/effective_dart_rules/analysis_options.yaml b/tool/dart_site/analysis_options.yaml similarity index 100% rename from tool/effective_dart_rules/analysis_options.yaml rename to tool/dart_site/analysis_options.yaml diff --git a/tool/dart_site/bin/dart_site.dart b/tool/dart_site/bin/dart_site.dart new file mode 100644 index 0000000000..ac237db4e7 --- /dev/null +++ b/tool/dart_site/bin/dart_site.dart @@ -0,0 +1,30 @@ +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:dart_site/dart_site.dart'; +import 'package:io/io.dart' as io; +import 'package:path/path.dart' as path; + +void main(List args) async { + // Verify that we are running from the root of the website repository. + if (!Directory(path.join('tool', 'dart_site')).existsSync()) { + throw Exception( + 'ERROR: Wrong directory, run from root of the repository.', + ); + } + + final runner = DartSiteCommandRunner(); + try { + final result = + await runner.run(args).whenComplete(io.sharedStdIn.terminate); + + exit(result is int ? result : 0); + } on UsageException catch (e) { + stderr.writeln(e); + exit(64); + } catch (e, stackTrace) { + stderr.writeln(e); + stderr.writeln(stackTrace); + exit(1); + } +} diff --git a/tool/dart_site/lib/dart_site.dart b/tool/dart_site/lib/dart_site.dart new file mode 100644 index 0000000000..0b092cf6e6 --- /dev/null +++ b/tool/dart_site/lib/dart_site.dart @@ -0,0 +1,29 @@ +import 'package:args/command_runner.dart'; + +import 'src/commands/analyze_dart.dart'; +import 'src/commands/check_all.dart'; +import 'src/commands/check_link_references.dart'; +import 'src/commands/check_links.dart'; +import 'src/commands/format_dart.dart'; +import 'src/commands/generate_effective_dart_toc.dart'; +import 'src/commands/refresh_excerpts.dart'; +import 'src/commands/test_dart.dart'; +import 'src/commands/verify_firebase_json.dart'; + +final class DartSiteCommandRunner extends CommandRunner { + DartSiteCommandRunner() + : super( + 'dart_site', + 'Infrastructure tooling for the Dart documentation website.', + ) { + addCommand(CheckLinksCommand()); + addCommand(CheckLinkReferencesCommand()); + addCommand(VerifyFirebaseJsonCommand()); + addCommand(RefreshExcerptsCommand()); + addCommand(FormatDartCommand()); + addCommand(GenerateEffectiveDartToc()); + addCommand(AnalyzeDartCommand()); + addCommand(TestDartCommand()); + addCommand(CheckAllCommand()); + } +} diff --git a/tool/dart_site/lib/src/commands/analyze_dart.dart b/tool/dart_site/lib/src/commands/analyze_dart.dart new file mode 100644 index 0000000000..67411fb253 --- /dev/null +++ b/tool/dart_site/lib/src/commands/analyze_dart.dart @@ -0,0 +1,75 @@ +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:path/path.dart' as path; + +import '../utils.dart'; + +final class AnalyzeDartCommand extends Command { + static const String _verboseFlag = 'verbose'; + + AnalyzeDartCommand() { + argParser.addFlag( + _verboseFlag, + defaultsTo: false, + help: 'Show verbose logging.', + ); + } + + @override + String get description => 'Run analysis on the site infra and examples.'; + + @override + String get name => 'analyze-dart'; + + @override + Future run() async => analyzeDart( + verboseLogging: argResults.get(_verboseFlag, false), + ); +} + +int analyzeDart({ + bool verboseLogging = false, +}) { + final directoriesToAnalyze = [ + path.join('tool', 'dart_site'), + ...dartProjectExampleDirectories, + ]; + + print('Analyzing code...'); + + for (final directory in directoriesToAnalyze) { + if (verboseLogging) { + print('Analyzing code in $directory...'); + } + + if (runPubGetIfNecessary(directory) case final pubGetResult + when pubGetResult != 0) { + return pubGetResult; + } + + final dartAnalyzeOutput = Process.runSync( + Platform.executable, + const ['analyze', '.'], + workingDirectory: directory, + ); + + if (dartAnalyzeOutput.exitCode != 0) { + final normalOutput = dartAnalyzeOutput.stdout.toString(); + final errorOutput = dartAnalyzeOutput.stderr.toString(); + + stderr.write(normalOutput); + stderr.write(errorOutput); + stderr.writeln('Error: Analysis on $directory failed.'); + return 1; + } else { + if (verboseLogging) { + print('Successfully analyzed code in $directory!'); + } + } + } + + print('No issues found while analyzing!'); + + return 0; +} diff --git a/tool/dart_site/lib/src/commands/check_all.dart b/tool/dart_site/lib/src/commands/check_all.dart new file mode 100644 index 0000000000..554de35191 --- /dev/null +++ b/tool/dart_site/lib/src/commands/check_all.dart @@ -0,0 +1,44 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:args/command_runner.dart'; + +import '../utils.dart'; + +final class CheckAllCommand extends Command { + @override + String get description => 'Run all site tests and verification.'; + + @override + String get name => 'check-all'; + + @override + Future run() async { + const verificationTasks = [ + ['format-dart', '--check'], + ['analyze-dart'], + ['test-dart'], + ['refresh-excerpts', '--fail-on-update'], + ['verify-firebase-json'], + ]; + + var seenFailure = false; + + for (final task in verificationTasks) { + groupStart(task.first); + final process = await Process.start( + Platform.executable, + ['run', 'dart_site', ...task], + ); + await stdout.addStream(process.stdout); + await stderr.addStream(process.stderr); + final processExitCode = await process.exitCode; + if (processExitCode != 0) { + seenFailure = true; + } + groupEnd(); + } + + return seenFailure ? 1 : 0; + } +} diff --git a/tool/dart_tools/bin/check_link_references.dart b/tool/dart_site/lib/src/commands/check_link_references.dart similarity index 61% rename from tool/dart_tools/bin/check_link_references.dart rename to tool/dart_site/lib/src/commands/check_link_references.dart index 3b908807f0..80d413340f 100644 --- a/tool/dart_tools/bin/check_link_references.dart +++ b/tool/dart_site/lib/src/commands/check_link_references.dart @@ -1,7 +1,99 @@ import 'dart:io'; +import 'package:args/command_runner.dart'; import 'package:path/path.dart' as path; +final class CheckLinkReferencesCommand extends Command { + @override + String get description => 'Verify there are no unlinked/broken ' + 'Markdown link references in the generated site output.'; + + @override + String get name => 'check-link-references'; + + @override + Future run() async => _checkLinkReferences(); +} + +int _checkLinkReferences() { + print('Checking for broken Markdown link references...'); + + const generatedSiteDirectory = '_site'; + + final directory = Directory(generatedSiteDirectory); + + if (!directory.existsSync()) { + stderr.writeln( + 'Error: Generated site not found at $generatedSiteDirectory. ' + 'Make sure the site is generated first!', + ); + return 1; + } + + final filesToInvalidReferences = _findInvalidLinkReferences(directory); + + if (filesToInvalidReferences.isNotEmpty) { + stderr.writeln('Error: Invalid link references found!'); + + filesToInvalidReferences.forEach((sourceFile, invalidReferences) { + stderr.writeln('\n$sourceFile:'); + for (final invalidReference in invalidReferences) { + stderr.writeln(invalidReference); + } + }); + + return 1; + } + + print('No invalid link references found.'); + + return 0; +} + +/// Find all invalid link references within generated HTML files +/// in the specified [directory]. +Map> _findInvalidLinkReferences(Directory directory) { + final invalidReferences = >{}; + + for (final filePath in directory + .listSync(recursive: true) + .map((f) => f.path) + .where((p) => path.extension(p) == '.html')) { + final content = File(filePath).readAsStringSync(); + final results = _findInContent(content); + if (results.isNotEmpty) { + invalidReferences[path.relative(filePath, from: directory.path)] = + results; + } + } + + return invalidReferences; +} + +List _findInContent(String content) { + for (final replacement in _allReplacements) { + content = content.replaceAll(replacement, ''); + } + + // Use regex to find all links that displayed abnormally, + // since a valid reference link should be an `` tag after rendered: + // + // - `[flutter.dev][]` + // - `[GitHub repo][repo]` + // See also: + // - https://github.github.com/gfm/#reference-link + final invalidFound = _invalidLinkReferencePattern.allMatches(content); + + if (invalidFound.isEmpty) { + return const []; + } + + return invalidFound + .map((e) => e[0]) + .whereType() + .toList(growable: false); +} + /// Ignore blocks with TODOs: /// /// ```html @@ -27,8 +119,9 @@ final _codeBlockPattern = RegExp(r'', dotAll: true); ///

/// ``` final _pullRequestTitlePattern = RegExp( - r'

\d+.*?

', - dotAll: true); + r'

\d+.*?

', + dotAll: true, +); /// Ignore PR titles that look like a link, /// directly embedded in a `
  • ` @@ -38,10 +131,12 @@ final _pullRequestTitlePattern = RegExp( ///
  • [docs][FWW] DropdownButton, ScaffoldMessenger, and StatefulBuilder links /// by @craiglabenz in https://github.com/flutter/flutter/pull/100316
  • /// ``` -final _pullRequestTitleInListItemPattern = - RegExp(r'
  • .*? in.*?https://github.com/.*?/pull/.*?
  • ', dotAll: true); +final _pullRequestTitleInListItemPattern = RegExp( + r'
  • .*? in.*?https://github.com/.*?/pull/.*?
  • ', + dotAll: true, +); -/// All replacements to run on file content before finding invalid references. +/// All replacements to run on a file content before finding invalid references. final _allReplacements = [ _htmlCommentPattern, _codeBlockPattern, @@ -50,64 +145,3 @@ final _allReplacements = [ ]; final _invalidLinkReferencePattern = RegExp(r'\[[^\[\]]+]\[[^\[\]]*]'); - -List _findInContent(String content) { - for (final replacement in _allReplacements) { - content = content.replaceAll(replacement, ''); - } - - // Use regex to find all links that displayed abnormally, - // since a valid reference link should be an `` tag after rendered: - // - // - `[flutter.dev][]` - // - `[GitHub repo][repo]` - // See also: - // - https://github.github.com/gfm/#reference-link - final invalidFound = _invalidLinkReferencePattern.allMatches(content); - - if (invalidFound.isEmpty) { - return const []; - } - - return invalidFound - .map((e) => e[0]) - .whereType() - .toList(growable: false); -} - -/// Find all invalid link references -/// within generated HTML files -/// in the specified [directory]. -Map> findInvalidLinkReferences(String directory) { - final invalidReferences = >{}; - - for (final filePath in Directory(directory) - .listSync(recursive: true) - .map((f) => f.path) - .where((p) => path.extension(p) == '.html')) { - final content = File(filePath).readAsStringSync(); - final results = _findInContent(content); - if (results.isNotEmpty) { - invalidReferences[path.relative(filePath, from: directory)] = results; - } - } - - return invalidReferences; -} - -void main() { - final filesToInvalidReferences = findInvalidLinkReferences('_site'); - - if (filesToInvalidReferences.isNotEmpty) { - print('check_link_references: Invalid link references found!'); - - filesToInvalidReferences.forEach((sourceFile, invalidReferences) { - print('\n$sourceFile:'); - for (final invalidReference in invalidReferences) { - print(invalidReference); - } - }); - - exit(1); - } -} diff --git a/tool/dart_site/lib/src/commands/check_links.dart b/tool/dart_site/lib/src/commands/check_links.dart new file mode 100644 index 0000000000..35afef49ca --- /dev/null +++ b/tool/dart_site/lib/src/commands/check_links.dart @@ -0,0 +1,124 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:linkcheck/linkcheck.dart' as linkcheck show run; +import 'package:path/path.dart' as path; + +import '../utils.dart'; + +final class CheckLinksCommand extends Command { + static const String _externalFlag = 'external'; + + CheckLinksCommand() { + argParser.addFlag( + _externalFlag, + abbr: 'e', + defaultsTo: false, + help: 'Verify external links as well.', + ); + } + + @override + String get description => 'Verify all links between pages on the site work.'; + + @override + String get name => 'check-links'; + + @override + Future run() async => _checkLinks( + checkExternal: argResults.get(_externalFlag, false), + ); +} + +/// The port that the firebase emulator runs on by default. +/// This must match what's declared in the `firebase.json` +/// and can't be 5000, since Airplay uses it. +const int _emulatorPort = 5500; + +/// The path from root where the linkcheck skip list lives. +final String _skipFilePath = path.join( + 'tool', + 'config', + 'linkcheck-skip-list.txt', +); + +Future _checkLinks({bool checkExternal = false}) async { + if (await _isPortInUse(_emulatorPort)) { + stderr.writeln( + 'Error: Port $_emulatorPort is already in use! ' + 'Are you running the emulator elsewhere?', + ); + return 1; + } + + print('Starting the Firebase hosting emulator asynchronously...'); + final emulatorProcess = await Process.start('npx', const [ + 'firebase', + 'emulators:start', + '--only', + 'hosting', + '--project', + 'default', + ]); + + // Ignore the stdin and stderr output from the emulator. + unawaited(emulatorProcess.stdout.drain()); + unawaited(emulatorProcess.stderr.drain()); + + // Give the emulator a few seconds to start up. + await Future.delayed(const Duration(seconds: 3)); + + try { + // Check to see if the emulator is running. + if (!(await _isPortInUse(_emulatorPort))) { + stderr.writeln('Error: The Firebase hosting emulator did not start!'); + return 1; + } + + try { + final result = await linkcheck.run( + [ + ':$_emulatorPort', + '--skip-file', + _skipFilePath, + if (checkExternal) 'external' + ], + stdout, + ); + return result; + } catch (e, stackTrace) { + stderr.writeln('Error: linkcheck failed to execute properly!'); + stderr.writeln(e); + stderr.writeln(stackTrace); + return 1; + } + } finally { + print('Shutting down Firebase hosting emulator...'); + emulatorProcess.kill(ProcessSignal.sigkill); + print('Done!\n'); + } +} + +/// If the specified [port] is in use. +Future _isPortInUse(int port) async { + try { + // Try to bind to the specified port. + final server = await ServerSocket.bind( + InternetAddress.loopbackIPv4, + port, + shared: false, + ).timeout(const Duration(seconds: 2)); // Ignore timeout. + + // If we reach this line, the port was available, + // and we know the Firebase hosting emulator is not running. + // So close the fake server and return as not in use. + await server.close(); + return false; + } on SocketException { + // If there is a socket exception, + // assume it is because the Firebase hosting emulator is already + // using the port. + return true; + } +} diff --git a/tool/dart_site/lib/src/commands/format_dart.dart b/tool/dart_site/lib/src/commands/format_dart.dart new file mode 100644 index 0000000000..5824fb9b9b --- /dev/null +++ b/tool/dart_site/lib/src/commands/format_dart.dart @@ -0,0 +1,68 @@ +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:path/path.dart' as path; + +import '../utils.dart'; + +final class FormatDartCommand extends Command { + static const String _checkFlag = 'check'; + + FormatDartCommand() { + argParser.addFlag( + _checkFlag, + defaultsTo: false, + help: 'Just check the formatting, do not update.', + ); + } + + @override + String get description => 'Format or check formatting of the site ' + 'examples and tools.'; + + @override + String get name => 'format-dart'; + + @override + Future run() async => formatDart( + justCheck: argResults.get(_checkFlag, false), + ); +} + +int formatDart({bool justCheck = false}) { + // Currently format all Dart files in the /tool directory + // and everything in /examples. + final directoriesToFormat = [ + 'tool', + ...Directory('examples') + .listSync() + .whereType() + .map((e) => e.path) + .where((e) => !path.basename(e).startsWith('.')), + ]; + + final dartFormatOutput = Process.runSync(Platform.resolvedExecutable, [ + 'format', + if (justCheck) ...['-o', 'none'], // Don't make changes if just checking. + ...directoriesToFormat, + ]); + + final normalOutput = dartFormatOutput.stdout.toString(); + final errorOutput = dartFormatOutput.stderr.toString(); + + stdout.write(normalOutput); + + if (dartFormatOutput.exitCode != 0) { + stderr.writeln('Error: Failed to run dart format:'); + stderr.write(errorOutput); + return 1; + } + + // If just checking formatting, exit with error code if any files changed. + if (justCheck && !normalOutput.contains('0 changed')) { + stderr.writeln('Error: Some files needed to be formatted!'); + return 1; + } + + return 0; +} diff --git a/tool/effective_dart_rules/bin/main.dart b/tool/dart_site/lib/src/commands/generate_effective_dart_toc.dart similarity index 50% rename from tool/effective_dart_rules/bin/main.dart rename to tool/dart_site/lib/src/commands/generate_effective_dart_toc.dart index 0c3ebae72a..0073b8f6af 100644 --- a/tool/effective_dart_rules/bin/main.dart +++ b/tool/dart_site/lib/src/commands/generate_effective_dart_toc.dart @@ -1,23 +1,51 @@ import 'dart:io'; +import 'package:args/command_runner.dart'; import 'package:html_unescape/html_unescape_small.dart'; import 'package:markdown/markdown.dart' as md; import 'package:path/path.dart' as path; -final _unescape = HtmlUnescape(); -final _anchorPattern = RegExp(r'(.+)\{#([^#]+)\}'); +import '../utils.dart'; -void main() async { +final HtmlUnescape _unescape = HtmlUnescape(); +final RegExp _anchorPattern = RegExp(r'(.+)\{#([^#]+)\}'); + +final class GenerateEffectiveDartToc extends Command { + static const String _checkFlag = 'check'; + + GenerateEffectiveDartToc() { + argParser.addFlag( + _checkFlag, + defaultsTo: false, + help: 'Just check if the TOC is up to date, do not update.', + ); + } + + @override + String get description => 'Generate or check up-to-date status of the ' + 'Effective Dart table of contents.'; + + @override + String get name => 'effective-dart'; + + @override + Future run() async => await _generateToc( + justCheck: argResults.get(_checkFlag, false), + ); +} + +Future _generateToc({bool justCheck = false}) async { const dirPath = 'src/effective-dart'; const filenames = ['style.md', 'documentation.md', 'usage.md', 'design.md']; final sections = - filenames.map((name) => Section(dirPath, name)).toList(growable: false); + filenames.map((name) => _Section(dirPath, name)).toList(growable: false); for (final section in sections) { - var lines = section.file.readAsLinesSync(); - // Ignore the YAML front matter (can lead to false H3 elements). - lines = lines + // Read the lines, but skip the YAML front matter, + // as it can lead to incorrect h3 elements. + final lines = section.file + .readAsLinesSync() .skip(1) .skipWhile((line) => line.trim() != '---') .toList(growable: false); @@ -26,52 +54,76 @@ void main() async { final nodes = document.parseLines(lines); for (final element in nodes.whereType()) { if (element.tag == 'h2') { - final subsection = Subsection(element); + final subsection = _Subsection(element); section.subsections.add(subsection); } else if (element.tag == 'h3') { - final rule = Rule(element); + final rule = _Rule(element); section.subsections.last.rules.add(rule); } } } - final outFile = File(path.join(dirPath, '_toc.md')); - IOSink? out; - try { - out = outFile.openWrite(); - - out.writeln(r''' + final newOutput = StringBuffer(); + newOutput.writeln(r''' {% comment %} This file is generated from the other files in this directory. To re-generate it, please run the following command from root of the project: ``` -$ dart run tool/effective_dart_rules/bin/main.dart +$ dart run dart_site effective-dart ``` {% endcomment %} '''); - out.writeln(r"
    "); - for (var i = 0; i < sections.length; i++) { - final section = sections[i]; - if (i > 0) { - if (i.isEven) { - out.writeln("
    "); - } - out.writeln( - "
    \n"); + newOutput.writeln( + r"
    ", + ); + + for (var sectionIndex = 0; sectionIndex < sections.length; sectionIndex++) { + final section = sections[sectionIndex]; + if (sectionIndex > 0) { + if (sectionIndex.isEven) { + newOutput.writeln("
    "); } - write(out, section); - out.writeln('\n
    '); + newOutput.writeln( + "
    \n", + ); } - out.writeln("
    "); - } finally { - await out?.close(); + _writeSection(newOutput, section); + newOutput.writeln('\n
    '); } + + newOutput.writeln("
    "); + + final tocFile = File(path.join(dirPath, '_toc.md')); + try { + final oldContents = tocFile.readAsStringSync(); + + if (oldContents != newOutput.toString()) { + if (justCheck) { + stderr.writeln( + 'Error: The Effective Dart TOC needs to be regenerated!', + ); + return 1; + } else { + tocFile.writeAsStringSync(newOutput.toString()); + print('Successfully updated the Effective Dart TOC.'); + } + } else { + print('The Effective Dart TOC is up to date!'); + } + } catch (e, stackTrace) { + stderr.writeln('Error: Failed to read or write the TOC file.'); + stderr.writeln(e); + stderr.writeln(stackTrace); + return 1; + } + + return 0; } -void write(IOSink out, Section section) { +void _writeSection(StringSink out, _Section section) { out.writeln('\n### ${section.name}\n'); for (final subsection in section.subsections) { out.writeln('\n**${subsection.name}**\n'); @@ -82,17 +134,18 @@ void write(IOSink out, Section section) { } } -class Rule { +class _Rule { final String html; final String fragment; - factory Rule(md.Element element) { + factory _Rule(md.Element element) { var name = _concatenatedText(element); var html = md.renderToHtml(element.children ?? const []); - var fragment = _generateAnchorHash(name); // Handle headers with an explicit "{#anchor-text}" anchor. var match = _anchorPattern.firstMatch(name); + + final String fragment; if (match != null) { // Pull the anchor from the name. name = (match[1] ?? '').trim(); @@ -103,40 +156,43 @@ class Rule { if (match != null) { html = (match[1] ?? '').trim(); } + } else { + fragment = _generateAnchorHash(name); } if (html.endsWith('.')) { throw Exception( - "Effective Dart rule '$name' ends with a period when it shouldn't."); + "Effective Dart rule '$name' ends with a period when it shouldn't.", + ); } html += '.'; - return Rule._(html, fragment); + return _Rule._(html, fragment); } - Rule._(this.html, this.fragment); + _Rule._(this.html, this.fragment); } -class Section { +class _Section { final Uri uri; final File file; final String name; - final List subsections = []; + final List<_Subsection> subsections = []; - Section(String dirPath, String filename) + _Section(String dirPath, String filename) : file = File(path.join(dirPath, filename)), uri = Uri.parse('/effective-dart/').resolve(filename.split('.').first), name = '${filename[0].toUpperCase()}' "${filename.substring(1).split('.').first}"; } -class Subsection { +class _Subsection { final String name; final String fragment; - final List rules = []; + final List<_Rule> rules = []; - Subsection(md.Element element) + _Subsection(md.Element element) : name = _concatenatedText(element), fragment = _generateAnchorHash(_concatenatedText(element)); } diff --git a/tool/dart_site/lib/src/commands/refresh_excerpts.dart b/tool/dart_site/lib/src/commands/refresh_excerpts.dart new file mode 100644 index 0000000000..967147f68e --- /dev/null +++ b/tool/dart_site/lib/src/commands/refresh_excerpts.dart @@ -0,0 +1,182 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:path/path.dart' as path; +import '../utils.dart'; + +final class RefreshExcerptsCommand extends Command { + static const String _verboseFlag = 'verbose'; + static const String _deleteCacheFlag = 'delete-cache'; + static const String _failOnUpdateFlag = 'fail-on-update'; + + RefreshExcerptsCommand() { + argParser.addFlag( + _verboseFlag, + defaultsTo: false, + help: 'Show verbose logging.', + ); + argParser.addFlag( + _deleteCacheFlag, + defaultsTo: false, + help: 'Delete dart build tooling and cache files after running.', + ); + argParser.addFlag( + _failOnUpdateFlag, + defaultsTo: false, + help: 'Fails if updates were needed.', + ); + } + + @override + String get description => 'Updates all code excerpts on the site.'; + + @override + String get name => 'refresh-excerpts'; + + @override + Future run() async => _refreshExcerpts( + verboseLogging: argResults.get(_verboseFlag, false), + deleteCache: argResults.get(_deleteCacheFlag, false), + failOnUpdate: argResults.get(_failOnUpdateFlag, false)); +} + +Future _refreshExcerpts({ + bool verboseLogging = false, + bool deleteCache = false, + bool failOnUpdate = false, +}) async { + final repositoryRoot = Directory.current.path; + final temporaryRoot = Directory.systemTemp.path; + final fragments = path.join(temporaryRoot, '_excerpter_fragments'); + + // Delete any existing fragments. + final fragmentsDirectory = Directory(fragments); + if (fragmentsDirectory.existsSync()) { + if (verboseLogging) { + print('Deleting previously generated $fragments.'); + } + fragmentsDirectory.deleteSync(recursive: true); + } + + print('Running the code excerpt fragment generator...'); + + // Run the code excerpter tool to generate the fragments used for updates. + final excerptsGenerated = Process.runSync(Platform.resolvedExecutable, [ + 'run', + 'build_runner', + 'build', + '--delete-conflicting-outputs', + '--config', + 'excerpt', + '--output', + fragments, + ]); + + if (verboseLogging) { + print(excerptsGenerated.stdout); + } + + // If the excerpt fragments were not generated successfully, + // then output the error log and return 1 to indicate failure. + if (excerptsGenerated.exitCode != 0) { + stderr.writeln('Error: Excerpt generation failed:'); + stderr.writeln(excerptsGenerated.stderr); + return 1; + } + + print('Code excerpt fragments generated successfully.'); + + // Verify the fragments directory for the /examples was generated properly. + if (!Directory(path.join(fragments, 'examples')).existsSync()) { + stderr.writeln( + 'Error: The examples fragments folder was not generated!', + ); + return 1; + } + + // A collection of replacements for the code excerpt updater tool + // to run by default. + // They must not contain (unencoded/unescaped) spaces. + const replacements = [ + // Allows use of //!
    to force a line break (against dart format) + r'/\/\/!
    //g;', + // Replace the word ellipsis, with optional parentheses. + r'/ellipsis(<\w+>)?(\(\))?;?/.../g;', + // Replace commented out ellipses: /*...*/ --> ... + r'/\/\*(\s*\.\.\.\s*)\*\//$1/g;', + // Replace brackets with commented out ellipses: {/*-...-*/} --> ... + r'/\{\/\*-(\s*\.\.\.\s*)-\*\/\}/$1/g;', + // Remove markers declaring an analysis issue or runtime error. + r'/\/\/!(analysis-issue|runtime-error)[^\n]*//g;', + ]; + + final srcDirectoryPath = path.join(repositoryRoot, 'src'); + final updaterArguments = [ + '--fragment-dir-path', + path.join(fragments, 'examples'), + '--src-dir-path', + 'examples', + if (verboseLogging) '--log-fine', + '--yaml', + '--no-escape-ng-interpolation', + '--replace=${replacements.join('')}', + '--write-in-place', + srcDirectoryPath, + ]; + + print('Running the code excerpt updater...'); + + // Open a try block so we can guarantee + // any temporary files are deleted. + try { + // Run the code excerpt updater tool to update the code excerpts + // in the /src directory. + final excerptsUpdated = Process.runSync(Platform.resolvedExecutable, [ + 'run', + 'code_excerpt_updater', + ...updaterArguments, + ]); + + final updateOutput = excerptsUpdated.stdout.toString(); + final updateErrors = excerptsUpdated.stderr.toString(); + + final bool success; + + // Inform the user if the updater failed, didn't need to make any updates, + // or successfully refreshed each excerpt. + if (excerptsUpdated.exitCode != 0 || updateErrors.contains('Error')) { + stderr.writeln('Error: Excerpt generation failed:'); + stderr.write(updateErrors); + success = false; + } else if (updateOutput.contains('0 out of')) { + if (verboseLogging) { + print(updateOutput); + } + print('All code excerpts are already up to date!'); + success = true; + } else { + stdout.write(updateOutput); + + if (failOnUpdate) { + stderr.writeln('Error: Some code excerpts needed to be updated!'); + success = false; + } else { + print('Code excerpts successfully refreshed!'); + success = true; + } + } + return success ? 0 : 1; + } finally { + // Clean up Dart build cache files if desired. + if (deleteCache) { + if (verboseLogging) { + print('Removing cached build files.'); + } + final dartBuildCache = Directory(path.join('.dart_tool', 'build')); + if (dartBuildCache.existsSync()) { + dartBuildCache.deleteSync(recursive: true); + } + } + } +} diff --git a/tool/dart_site/lib/src/commands/test_dart.dart b/tool/dart_site/lib/src/commands/test_dart.dart new file mode 100644 index 0000000000..9ef5010bc1 --- /dev/null +++ b/tool/dart_site/lib/src/commands/test_dart.dart @@ -0,0 +1,87 @@ +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:path/path.dart' as path; + +import '../utils.dart'; + +final class TestDartCommand extends Command { + static const String _verboseFlag = 'verbose'; + + TestDartCommand() { + argParser.addFlag( + _verboseFlag, + defaultsTo: false, + help: 'Show verbose logging.', + ); + } + + @override + String get description => 'Run tests on the site infra and examples.'; + + @override + String get name => 'test-dart'; + + @override + Future run() async => _testDart( + verboseLogging: argResults.get(_verboseFlag, false), + ); +} + +int _testDart({ + bool verboseLogging = false, +}) { + final directoriesToTest = [ + path.join('tool', 'dart_site'), + ...dartProjectExampleDirectories, + ]; + + print('Testing code...'); + + for (final directory in directoriesToTest) { + if (verboseLogging) { + print('Testing code in $directory...'); + } + + if (runPubGetIfNecessary(directory) case final pubGetResult + when pubGetResult != 0) { + return pubGetResult; + } + + final dartTestOutput = Process.runSync( + Platform.executable, + const [ + 'test', + '--reporter', + 'expanded', // Non-animated expanded output looks better in CI and logs. + ], + workingDirectory: directory, + ); + + if (dartTestOutput.exitCode != 0) { + final normalOutput = dartTestOutput.stdout.toString(); + final errorOutput = dartTestOutput.stderr.toString(); + + // It's ok if the test directory is not found. + if (!errorOutput.contains('No test files were') && + !normalOutput.contains('Could not find package `test`')) { + stderr.write(normalOutput); + stderr.writeln('Error: Tests in $directory failed:'); + stderr.write(errorOutput); + return 1; + } + + if (verboseLogging) { + print('No tests found or ran in $directory.'); + } + } else { + if (verboseLogging) { + print('All tests passed in $directory.'); + } + } + } + + print('All tests passed successfully!'); + + return 0; +} diff --git a/tool/dart_site/lib/src/commands/verify_firebase_json.dart b/tool/dart_site/lib/src/commands/verify_firebase_json.dart new file mode 100644 index 0000000000..513afc15ce --- /dev/null +++ b/tool/dart_site/lib/src/commands/verify_firebase_json.dart @@ -0,0 +1,152 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:args/command_runner.dart'; + +final class VerifyFirebaseJsonCommand extends Command { + @override + String get description => 'Verify the firebase.json file is valid and ' + 'meets the site standards.'; + + @override + String get name => 'verify-firebase-json'; + + @override + Future run() async => _verifyFirebaseJson(); +} + +int _verifyFirebaseJson() { + final firebaseFile = File('firebase.json'); + + if (!firebaseFile.existsSync()) { + stderr.writeln( + 'Cannot find the firebase.json file in the current directory.', + ); + return 1; + } + + try { + final firebaseConfigString = firebaseFile.readAsStringSync(); + final firebaseConfig = + jsonDecode(firebaseConfigString) as Map; + + final hostingConfig = firebaseConfig['hosting'] as Map?; + + if (hostingConfig == null) { + stderr.writeln( + "Error: The firebase.json file is missing a top-level 'hosting' entry.", + ); + return 1; + } + + final redirects = hostingConfig['redirects']; + + if (redirects == null) { + stdout.writeln( + 'There are no redirects specified within the firebase.json file.', + ); + return 0; + } + + if (redirects is! List) { + stderr.writeln( + "Error: The firebase.json file's 'redirect' entry is not a list.", + ); + return 1; + } + + if (redirects.isEmpty) { + return 0; + } + + final sources = {}; + + var duplicatesFound = 0; + + for (final redirect in redirects) { + if (redirect is! Map) { + stderr.writeln( + 'Error: Each redirect must be a map containing ' + "a 'source' or 'regex' field.", + ); + return 1; + } + + final source = redirect['source'] ?? redirect['regex']; + if (source == null) { + stderr.writeln( + 'Error: The firebase.json file has a ' + "redirect missing a 'source' or 'regex'.", + ); + return 1; + } + + if (source is! String) { + stderr.writeln( + 'Error: The firebase.json redirect $redirect has a ' + "'source' or 'regex' specified which is not a string.", + ); + return 1; + } + + if (source.isEmpty) { + stderr.writeln( + 'Error: The firebase.json redirect $redirect has an ' + "empty 'source' or 'regex'.", + ); + return 1; + } + + if (sources.contains(source)) { + stderr.writeln( + "ERROR: Multiple redirects share the '$source' source.", + ); + duplicatesFound += 1; + } + + sources.add(source); + + final destination = redirect['destination']; + + if (destination == null) { + stderr.writeln( + 'Error: The firebase.json file has a ' + "redirect missing a 'destination'.", + ); + return 1; + } + + if (destination is! String) { + stderr.writeln( + 'Error: The firebase.json redirect $redirect has a ' + "'destination' specified which is not a string.", + ); + return 1; + } + + if (destination.isEmpty) { + stderr.writeln( + 'Error: The firebase.json redirect $redirect has ' + "an empty 'destination'.", + ); + return 1; + } + } + + if (duplicatesFound > 0) { + stderr.writeln( + 'Error: $duplicatesFound duplicate sources found ' + 'in the firebase.json redirects.', + ); + return 1; + } + } catch (e) { + stderr.writeln( + 'Error: Encountered an error when loading the firebase.json file:', + ); + stderr.writeln(e); + return 1; + } + + return 0; +} diff --git a/tool/dart_site/lib/src/utils.dart b/tool/dart_site/lib/src/utils.dart new file mode 100644 index 0000000000..6e73401569 --- /dev/null +++ b/tool/dart_site/lib/src/utils.dart @@ -0,0 +1,88 @@ +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:path/path.dart' as path; + +final bool _runningInCi = Platform.environment['CI'] == 'true'; + +void groupStart(String text) { + if (_runningInCi) { + print('::group::$text'); + } else { + print(''); + } +} + +void groupEnd() { + if (_runningInCi) { + print('::endgroup::'); + } +} + +int runPubGetIfNecessary(String directory) { + final pubGetOutput = Process.runSync( + Platform.executable, + const ['pub', 'get'], + workingDirectory: directory, + ); + + if (pubGetOutput.exitCode != 0) { + final normalOutput = pubGetOutput.stdout.toString(); + final errorOutput = pubGetOutput.stderr.toString(); + + stderr.write(normalOutput); + stderr.write(errorOutput); + stderr.writeln('Error: Pub get in $directory failed.'); + return 1; + } + + return 0; +} + +extension ArgResultExtensions on ArgResults? { + T get(String key, T defaultValue) => this?[key] as T? ?? defaultValue; +} + +/// A collection of the paths of all Dart projects with +/// a pubspec.yaml file in the `/examples` directory, +/// excluding ones in hidden directories or codelabs. +final List dartProjectExampleDirectories = + findNestedDirectoriesWithPubspec( + Directory('examples'), + skipPaths: {}, + skipHidden: true, +)..sort(); + +List findNestedDirectoriesWithPubspec( + Directory rootDirectory, { + Set skipPaths = const {}, + bool skipHidden = true, +}) { + final normalizedPath = path.normalize(rootDirectory.path); + + // Base case: Doesn't exist, skipped, or hidden. + if (skipPaths.contains(normalizedPath) || + (skipHidden && path.basename(normalizedPath).startsWith('.')) || + !rootDirectory.existsSync()) { + return const []; + } + + final directoriesWithPubspec = []; + + for (final entity in rootDirectory.listSync()) { + if (entity is Directory) { + // If this entity is a direct, recurse in to it + // to find any pubspec files. + directoriesWithPubspec.addAll(findNestedDirectoriesWithPubspec( + entity, + skipPaths: skipPaths, + skipHidden: skipHidden, + )); + } else if (entity is File && path.basename(entity.path) == 'pubspec.yaml') { + // If the directory has a pubspec.yaml file, this directory counts. + directoriesWithPubspec.add(normalizedPath); + } + } + + return directoriesWithPubspec; +} diff --git a/tool/effective_dart_rules/pubspec.yaml b/tool/dart_site/pubspec.yaml similarity index 53% rename from tool/effective_dart_rules/pubspec.yaml rename to tool/dart_site/pubspec.yaml index e7bc067b18..48780be5e6 100644 --- a/tool/effective_dart_rules/pubspec.yaml +++ b/tool/dart_site/pubspec.yaml @@ -1,15 +1,17 @@ -name: effective_dart_rules -description: > - Builds a file with all the rules in the different sections of Effective Dart. +name: dart_site +description: Dart-based tools for building dart.dev. publish_to: none environment: sdk: ^3.2.0 dependencies: + args: ^2.4.2 html_unescape: ^2.0.0 - markdown: ^7.1.1 - path: ^1.8.3 + io: ^1.0.4 + linkcheck: ^3.0.0 + markdown: ^7.2.0 + path: ^1.9.0 dev_dependencies: analysis_defaults: diff --git a/tool/effective_dart_rules/.gitignore b/tool/effective_dart_rules/.gitignore deleted file mode 100644 index 2de1a04141..0000000000 --- a/tool/effective_dart_rules/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Files and directories created by pub -.packages -.pub/ -packages -pubspec.lock # (Remove this pattern if you wish to check in your lock file) diff --git a/tool/fetch-dart-sdk-sums.sh b/tool/fetch-dart-sdk-sums.sh deleted file mode 100755 index 7851493201..0000000000 --- a/tool/fetch-dart-sdk-sums.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash -# Use this file locally to update Dart SDK checksum values in the Dockerfile -# Prints output similar to cases in Dockerfile for easy composition -# when having to update checksum values for updates dart SDK. -set -eu -o pipefail -TOOL_DIR="${TOOL_DIR:=$(dirname "$0")}" -source $TOOL_DIR/utils.sh - -VERSION="latest" -CHANNEL="stable" - -while (( "$#" )); do - case "$1" in - --version) - VERSION=$2 - shift 2 - ;; - --channel) - CHANNEL=$2 - shift 2 - ;; - *) - echo "Unsupported argument $1" >&2 - exit 1 - ;; - esac -done - -BASEURL="https://storage.googleapis.com/dart-archive/channels" -CHANNELS="stable beta dev" -ARCHS="amd64 arm64" -ENDING='\\\n' - -printf "\n$(blue "Copy the following output and replace the existing code in the Dockerfile")\n" -printf "$(blue "inside the 'set -eu' run statement:")\n\n" - -for CHANNEL in $CHANNELS; do - for ARCH in $ARCHS; do - printf "$(yellow "${ARCH}_${CHANNEL})") $ENDING" - _arch=$ARCH - if [[ "$_arch" == "amd64" ]]; then - _arch='x64' - fi - _filename="dartsdk-linux-${_arch}-release.zip" - _url="$BASEURL/$CHANNEL/release/$VERSION/sdk/$_filename" - curl -fsSLO $_url - _checksum=$(shasum -a 256 $_filename) - read -a _fname_arr <<< "${_checksum}" # Read in string output as array - _checkonly="${_fname_arr%:*}" # Remove filename portion of checksum output - printf "$(yellow " DART_SHA256=\"$_fname_arr\";") $ENDING" - printf "$(yellow " SDK_ARCH=\"$_arch\";;") $ENDING" - rm $_filename - done -done - -echo "" diff --git a/tool/refresh-code-excerpts.sh b/tool/refresh-code-excerpts.sh deleted file mode 100755 index b04610e3d5..0000000000 --- a/tool/refresh-code-excerpts.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env bash -# Refresh all code excerpts - -set -eu -o pipefail -source $TOOL_DIR/utils.sh - - -EXAMPLES="$BASE_DIR/examples" -TMP="$BASE_DIR/tmp" -FRAG="$TMP/_fragments" -LOG_FILE="$TMP/refresh-code-excerpts.log" -KEEP_CACHE=0 -ARGS="" - -# Check only -|-- flags/args, save final optional positional (target source) -while [[ $# -gt 0 ]]; do - case "$1" in - -k|--keep-dart-tool) - KEEP_CACHE=1 - shift - ;; - --log-fine) - ARGS+="--log-fine " - shift - ;; - -h|--help) - echo "Usage: $(basename $0) [-h|--help] [-k|--keep-dart-tool] [target-source]" - exit - ;; - *) - echo "Unsupported argument $1" >&2 - exit 1 - ;; - esac -done - -printf "\n$(blue "Preparing excerpt refresh...")\n" - -# Final arg, if any, should be target or default to main /src -TARGET_SRC="${1:-$BASE_DIR/src}" - -if [[ ! -e $TARGET_SRC ]]; then - echo -e "$(red "Target source ($TARGET_SRC) does not exist")" - exit 1 -fi - -if [[ -e "$FRAG" ]]; then - echo -e "$(blue "Deleting existing fragments ($FRAG)")" - rm -rf $FRAG -fi - -if [[ ! -e "pubspec.lock" ]]; then - dart pub get -fi - -( set -x - dart run build_runner build \ - --delete-conflicting-outputs \ - --config excerpt \ - --output $FRAG) - -if [[ ! -e "$FRAG/examples" ]]; then - echo -e "$(red "Fragments directory ($FRAG/examples) was not generated")" - exit 1 -fi - -ARGS+="--yaml --no-escape-ng-interpolation --write-in-place " -ARGS+="--fragment-dir-path=$FRAG/examples " -ARGS+="--src-dir-path=$EXAMPLES " -ARGS+="--replace=" - -# The replace expressions that follow must not contain (unencode/unescaped) spaces: -ARGS+='/\/\/!
    //g;' # Use //!
    to force a line break (against dart format) -ARGS+='/ellipsis(<\w+>)?(\(\))?;?/.../g;' # ellipses; --> ... -ARGS+='/\/\*(\s*\.\.\.\s*)\*\//$1/g;' # /*...*/ --> ... -ARGS+='/\{\/\*-(\s*\.\.\.\s*)-\*\/\}/$1/g;' # {/*-...-*/} --> ... (removed brackets too) - -# Replace "//!analysis-issue" by, say, "Analysis issue" (although -# once we can use embedded DPs this won't be needed)? -ARGS+="/\/\/!(analysis-issue|runtime-error)[^\n]*//g;" # Removed warning/error marker - -printf "\n$(blue "Running excerpt refresh...")\n" -IFS=' '; read -a debug_args <<< $ARGS -for arg in "${debug_args[@]}" ; do - echo -e " $(yellow $arg)" -done -echo "--" - -# Script lives in: site-shared/packages/code_excerpt_updater -dart run code_excerpt_updater $ARGS $TARGET_SRC 2>&1 | tee $LOG_FILE - -if [[ -z "$KEEP_CACHE" ]]; then - echo -e "$(blue "Removing dart cache files...")" - rm -r "$BASE_DIR/.dart_tool/" -fi - -if [[ ! -f $LOG_FILE ]]; then - printf "\n$(red "Log file ($LOG_FILE) does not exist - something went wrong")\n\n" -fi - -LOGS=$(cat $LOG_FILE) -if [[ $LOGS != *" 0 out of"* || $LOGS == *Error* ]]; then - printf "\n$(red "Errors were encountered refreshing code excerpts")\n\n" - exit 1 -else - printf "\n$(blue "Looking good!")\n\n" -fi diff --git a/tool/test.sh b/tool/test.sh deleted file mode 100755 index 7cfa86ffd5..0000000000 --- a/tool/test.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# Run all tests for validating code examples -set -u - -source $TOOL_DIR/utils.sh - -EXIT_STATUS=0 - -tool/check-formatting.sh || EXIT_STATUS=1 -tool/refresh-code-excerpts.sh || EXIT_STATUS=1 -tool/analyze-and-test-examples.sh || EXIT_STATUS=1 - -printf "\n-- Done --\n\n" - -if [[ "$EXIT_STATUS" -eq "0" ]]; then - printf "\n$(blue "All tests and checks have passed!")\n\n" -else - printf "\n$(red " -Some tests and/or checks have failed. -Look at the full output from this script for details. - ")\n" -fi - -exit $EXIT_STATUS diff --git a/tool/update-dart-sums.sh b/tool/update-dart-sums.sh deleted file mode 100755 index bd2333f702..0000000000 --- a/tool/update-dart-sums.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -# Use this file to update the Dart SHA256 hashes -# in the Dockerfile. Run after check-dart-sdk.sh - -lead='# BEGIN dart-sha$' -tail='# END dart-sha$' -new_file='tool/new-dart-hashes.txt' -existing_file='Dockerfile' - -new_hash=$(sed -n -e '/DART_SHA/ p' -e '/DART_SHA/ q' $new_file) -old_hash=$(sed -n -e '/DART_SHA/ p' -e '/DART_SHA/ q' $existing_file) - -echo -e "Old $old_hash" -echo -e "New $new_hash" - -if [[ -z "$new_hash" ]]; then - echo "No new hash found." -else [[ -z "$old_hash" ]] - echo "Comparing hashes" - if [["$new_hash" = "$old_hash"]]; then - echo "Hashes match. No changes needed." - else - echo "New hashes found. Replacing hashes.\n" - echo $(sed -i.save -e "/$lead/,/$tail/{ /$lead/{p; r $new_file - }; /$tail/p;d;}" $existing_file) - echo "Replaced hashes" - fi -fi - -rm $new_file -echo -e "Removed $new_file. Re-run check-dart-sdk.sh to pull the current SHA hashes.\n" - -echo -e "Update completed." diff --git a/tool/utils.sh b/tool/utils.sh deleted file mode 100755 index 38e5059a06..0000000000 --- a/tool/utils.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -RED='\033[0;31m' -YELLOW='\033[0;33m' -BLUE='\033[0;34m' -GRAY='\033[0;37m' -END='\033[0m' - - -function blue() { - echo -e "$BLUE$1$END" -} - -function red() { - echo -e "$RED$1$END" -} - -function yellow() { - echo -e "$YELLOW$1$END" -} - -function base_dir() { - echo $( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -} From 9d54bc0fe3ca0a565dd8b61bca260615650eb3f9 Mon Sep 17 00:00:00 2001 From: Parker Lougheed Date: Wed, 17 Jan 2024 22:40:56 -0600 Subject: [PATCH 2/5] Add license headers --- tool/dart_site/bin/dart_site.dart | 4 ++++ tool/dart_site/lib/dart_site.dart | 4 ++++ tool/dart_site/lib/src/commands/analyze_dart.dart | 4 ++++ tool/dart_site/lib/src/commands/check_all.dart | 4 ++++ tool/dart_site/lib/src/commands/check_link_references.dart | 4 ++++ tool/dart_site/lib/src/commands/check_links.dart | 4 ++++ tool/dart_site/lib/src/commands/format_dart.dart | 4 ++++ .../lib/src/commands/generate_effective_dart_toc.dart | 4 ++++ tool/dart_site/lib/src/commands/refresh_excerpts.dart | 4 ++++ tool/dart_site/lib/src/commands/test_dart.dart | 4 ++++ tool/dart_site/lib/src/commands/verify_firebase_json.dart | 4 ++++ tool/dart_site/lib/src/utils.dart | 4 ++++ 12 files changed, 48 insertions(+) diff --git a/tool/dart_site/bin/dart_site.dart b/tool/dart_site/bin/dart_site.dart index ac237db4e7..72afb15611 100644 --- a/tool/dart_site/bin/dart_site.dart +++ b/tool/dart_site/bin/dart_site.dart @@ -1,3 +1,7 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + import 'dart:io'; import 'package:args/command_runner.dart'; diff --git a/tool/dart_site/lib/dart_site.dart b/tool/dart_site/lib/dart_site.dart index 0b092cf6e6..343a96a4c9 100644 --- a/tool/dart_site/lib/dart_site.dart +++ b/tool/dart_site/lib/dart_site.dart @@ -1,3 +1,7 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + import 'package:args/command_runner.dart'; import 'src/commands/analyze_dart.dart'; diff --git a/tool/dart_site/lib/src/commands/analyze_dart.dart b/tool/dart_site/lib/src/commands/analyze_dart.dart index 67411fb253..73b2bffa74 100644 --- a/tool/dart_site/lib/src/commands/analyze_dart.dart +++ b/tool/dart_site/lib/src/commands/analyze_dart.dart @@ -1,3 +1,7 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + import 'dart:io'; import 'package:args/command_runner.dart'; diff --git a/tool/dart_site/lib/src/commands/check_all.dart b/tool/dart_site/lib/src/commands/check_all.dart index 554de35191..8c9d5d8aa2 100644 --- a/tool/dart_site/lib/src/commands/check_all.dart +++ b/tool/dart_site/lib/src/commands/check_all.dart @@ -1,3 +1,7 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + import 'dart:async'; import 'dart:io'; diff --git a/tool/dart_site/lib/src/commands/check_link_references.dart b/tool/dart_site/lib/src/commands/check_link_references.dart index 80d413340f..f8a10281ab 100644 --- a/tool/dart_site/lib/src/commands/check_link_references.dart +++ b/tool/dart_site/lib/src/commands/check_link_references.dart @@ -1,3 +1,7 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + import 'dart:io'; import 'package:args/command_runner.dart'; diff --git a/tool/dart_site/lib/src/commands/check_links.dart b/tool/dart_site/lib/src/commands/check_links.dart index 35afef49ca..d53c43cba5 100644 --- a/tool/dart_site/lib/src/commands/check_links.dart +++ b/tool/dart_site/lib/src/commands/check_links.dart @@ -1,3 +1,7 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + import 'dart:async'; import 'dart:io'; diff --git a/tool/dart_site/lib/src/commands/format_dart.dart b/tool/dart_site/lib/src/commands/format_dart.dart index 5824fb9b9b..f3563462a4 100644 --- a/tool/dart_site/lib/src/commands/format_dart.dart +++ b/tool/dart_site/lib/src/commands/format_dart.dart @@ -1,3 +1,7 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + import 'dart:io'; import 'package:args/command_runner.dart'; diff --git a/tool/dart_site/lib/src/commands/generate_effective_dart_toc.dart b/tool/dart_site/lib/src/commands/generate_effective_dart_toc.dart index 0073b8f6af..6ac64fb389 100644 --- a/tool/dart_site/lib/src/commands/generate_effective_dart_toc.dart +++ b/tool/dart_site/lib/src/commands/generate_effective_dart_toc.dart @@ -1,3 +1,7 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + import 'dart:io'; import 'package:args/command_runner.dart'; diff --git a/tool/dart_site/lib/src/commands/refresh_excerpts.dart b/tool/dart_site/lib/src/commands/refresh_excerpts.dart index 967147f68e..9dcc3f7114 100644 --- a/tool/dart_site/lib/src/commands/refresh_excerpts.dart +++ b/tool/dart_site/lib/src/commands/refresh_excerpts.dart @@ -1,3 +1,7 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + import 'dart:async'; import 'dart:io'; diff --git a/tool/dart_site/lib/src/commands/test_dart.dart b/tool/dart_site/lib/src/commands/test_dart.dart index 9ef5010bc1..381a73498a 100644 --- a/tool/dart_site/lib/src/commands/test_dart.dart +++ b/tool/dart_site/lib/src/commands/test_dart.dart @@ -1,3 +1,7 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + import 'dart:io'; import 'package:args/command_runner.dart'; diff --git a/tool/dart_site/lib/src/commands/verify_firebase_json.dart b/tool/dart_site/lib/src/commands/verify_firebase_json.dart index 513afc15ce..6381751fcb 100644 --- a/tool/dart_site/lib/src/commands/verify_firebase_json.dart +++ b/tool/dart_site/lib/src/commands/verify_firebase_json.dart @@ -1,3 +1,7 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + import 'dart:convert'; import 'dart:io'; diff --git a/tool/dart_site/lib/src/utils.dart b/tool/dart_site/lib/src/utils.dart index 6e73401569..7f40c1377d 100644 --- a/tool/dart_site/lib/src/utils.dart +++ b/tool/dart_site/lib/src/utils.dart @@ -1,3 +1,7 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + import 'dart:io'; import 'package:args/args.dart'; From 3f0281ec2551be9934082ed0b593c34c20ac1382 Mon Sep 17 00:00:00 2001 From: Parker Lougheed Date: Wed, 17 Jan 2024 22:51:10 -0600 Subject: [PATCH 3/5] Provide more descriptive names for actions steps --- .github/workflows/test.yml | 27 ++++++++++++------- .../dart_site/lib/src/commands/test_dart.dart | 3 ++- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 05478c0992..cee06bede7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,11 +43,16 @@ jobs: - uses: dart-lang/setup-dart@ca7e6fee45ffbd82b555a7ebfc236d2c86439f5b with: sdk: ${{ matrix.sdk }} - - run: dart pub get - - run: dart run dart_site format-dart --check - - run: dart run dart_site analyze-dart - - run: dart run dart_site test-dart - - run: dart run dart_site refresh-excerpts --fail-on-update + - name: Fetch packages + run: dart pub get + - name: Check Dart code formatting + run: dart run dart_site format-dart --check + - name: Analyze Dart code + run: dart run dart_site analyze-dart + - name: Run Dart tests + run: dart run dart_site test-dart + - name: Check if excerpts are up to date + run: dart run dart_site refresh-excerpts --fail-on-update linkcheck: name: Build site and check links @@ -62,11 +67,15 @@ jobs: - uses: dart-lang/setup-dart@ca7e6fee45ffbd82b555a7ebfc236d2c86439f5b with: sdk: stable - - run: dart pub get - - run: dart run dart_site check-link-references - - run: dart run dart_site verify-firebase-json + - name: Fetch packages + run: dart pub get + - name: Check for broken Markdown links + run: dart run dart_site check-link-references + - name: Validate the firebase.json file + run: dart run dart_site verify-firebase-json - uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 with: node-version: ${{ env.NODE_VERSION }} - run: npm install -g firebase-tools@13.0.2 - - run: dart run dart_site check-links + - name: Check internal site links are functional + run: dart run dart_site check-links diff --git a/tool/dart_site/lib/src/commands/test_dart.dart b/tool/dart_site/lib/src/commands/test_dart.dart index 381a73498a..1fb6ba962a 100644 --- a/tool/dart_site/lib/src/commands/test_dart.dart +++ b/tool/dart_site/lib/src/commands/test_dart.dart @@ -68,7 +68,8 @@ int _testDart({ // It's ok if the test directory is not found. if (!errorOutput.contains('No test files were') && - !normalOutput.contains('Could not find package `test`')) { + !normalOutput.contains('Could not find package `test`') && + !normalOutput.contains('No tests were')) { stderr.write(normalOutput); stderr.writeln('Error: Tests in $directory failed:'); stderr.write(errorOutput); From 625a76ec48a6c3a0c946ba8d8c8fa3dc093a0ebe Mon Sep 17 00:00:00 2001 From: Parker Lougheed Date: Wed, 17 Jan 2024 22:57:45 -0600 Subject: [PATCH 4/5] Further cleanup of the tool directory --- tool/dart_site/bin/dart_site.dart | 2 +- .../dart_site/lib/src/commands/test_dart.dart | 2 +- .../src/commands/verify_firebase_json.dart | 2 +- tool/dart_tools/README.md | 16 ----- tool/dart_tools/analysis_options.yaml | 1 - tool/dart_tools/bin/update_analyzer_txt.dart | 65 ------------------- tool/dart_tools/pubspec.yaml | 14 ---- 7 files changed, 3 insertions(+), 99 deletions(-) delete mode 100644 tool/dart_tools/README.md delete mode 100644 tool/dart_tools/analysis_options.yaml delete mode 100644 tool/dart_tools/bin/update_analyzer_txt.dart delete mode 100644 tool/dart_tools/pubspec.yaml diff --git a/tool/dart_site/bin/dart_site.dart b/tool/dart_site/bin/dart_site.dart index 72afb15611..d066590a74 100644 --- a/tool/dart_site/bin/dart_site.dart +++ b/tool/dart_site/bin/dart_site.dart @@ -13,7 +13,7 @@ void main(List args) async { // Verify that we are running from the root of the website repository. if (!Directory(path.join('tool', 'dart_site')).existsSync()) { throw Exception( - 'ERROR: Wrong directory, run from root of the repository.', + 'Error: Wrong directory, run from root of the repository.', ); } diff --git a/tool/dart_site/lib/src/commands/test_dart.dart b/tool/dart_site/lib/src/commands/test_dart.dart index 1fb6ba962a..9da973424f 100644 --- a/tool/dart_site/lib/src/commands/test_dart.dart +++ b/tool/dart_site/lib/src/commands/test_dart.dart @@ -67,7 +67,7 @@ int _testDart({ final errorOutput = dartTestOutput.stderr.toString(); // It's ok if the test directory is not found. - if (!errorOutput.contains('No test files were') && + if (!errorOutput.contains('No test') && !normalOutput.contains('Could not find package `test`') && !normalOutput.contains('No tests were')) { stderr.write(normalOutput); diff --git a/tool/dart_site/lib/src/commands/verify_firebase_json.dart b/tool/dart_site/lib/src/commands/verify_firebase_json.dart index 6381751fcb..c799496534 100644 --- a/tool/dart_site/lib/src/commands/verify_firebase_json.dart +++ b/tool/dart_site/lib/src/commands/verify_firebase_json.dart @@ -103,7 +103,7 @@ int _verifyFirebaseJson() { if (sources.contains(source)) { stderr.writeln( - "ERROR: Multiple redirects share the '$source' source.", + "Error: Multiple redirects share the '$source' source.", ); duplicatesFound += 1; } diff --git a/tool/dart_tools/README.md b/tool/dart_tools/README.md deleted file mode 100644 index 227e107e07..0000000000 --- a/tool/dart_tools/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# dart_tools - -dart tools for site-www. - -## Usage - update_analyzer_txt.dart - -Make sure the Dart SDK in your PATH is the channel you want to use. -This script will update files matching analyzer-results-CHANNEL.txt -in the examples/ directories - -``` -cd tool/dart_tools -dart pub get -dart run bin/update_analyzer_txt.dart -``` - diff --git a/tool/dart_tools/analysis_options.yaml b/tool/dart_tools/analysis_options.yaml deleted file mode 100644 index b9bdf805ac..0000000000 --- a/tool/dart_tools/analysis_options.yaml +++ /dev/null @@ -1 +0,0 @@ -include: package:analysis_defaults/analysis.yaml diff --git a/tool/dart_tools/bin/update_analyzer_txt.dart b/tool/dart_tools/bin/update_analyzer_txt.dart deleted file mode 100644 index 8ddbf758eb..0000000000 --- a/tool/dart_tools/bin/update_analyzer_txt.dart +++ /dev/null @@ -1,65 +0,0 @@ -import 'dart:io'; -import 'package:path/path.dart' as path; - -void main(List args) async { - final channel = await getDartChannel(); - print('updating analyzer text using $channel channel SDK.'); - - final currentDirectoryComponents = path.split(Directory.current.path); - final siteRoot = path.joinAll(currentDirectoryComponents.sublist( - 0, currentDirectoryComponents.length - 2)); - - final examplesDir = Directory(path.absolute(path.join(siteRoot, 'examples'))); - final examples = await examplesDir - .list() - .where((event) => event is Directory) - .cast() - .toList(); - - for (final example in examples) { - await writeAnalyzerResultsFile(example, channel); - } -} - -Future writeAnalyzerResultsFile( - Directory directory, String channel) async { - print('Running "pub get" in ${directory.path}'); - await Process.run('dart', ['pub', 'get'], workingDirectory: directory.path); - print('Running "dart analyze" in ${directory.path}'); - - final output = - await Process.run('dart', ['analyze'], workingDirectory: directory.path); - - if (output.exitCode == 0) { - print('no analyzer results to write.'); - return; - } - if (output.exitCode != 3) { - throw Exception( - 'Unexpected exit code: ${output.exitCode}\n${output.stderr}'); - } - - var analyzerFile = - File(path.join(directory.path, 'analyzer-results-$channel.txt')); - if (!await analyzerFile.exists()) { - analyzerFile = File(path.join(directory.path, 'analyzer-results.txt')); - } - - print('writing results to ${analyzerFile.path}'); - final stdout = output.stdout; - if (stdout is! String) { - return; - } - await analyzerFile.writeAsString(stdout); -} - -Future getDartChannel() async { - final version = await Process.run('dart', ['--version']); - final regex = RegExp(r'version: \S+ \(([a-z]+)\)'); - final stderr = version.stderr; - if (stderr is! String) { - return 'missing'; - } - final match = regex.firstMatch(stderr); - return match?.group(1) ?? 'missing'; -} diff --git a/tool/dart_tools/pubspec.yaml b/tool/dart_tools/pubspec.yaml deleted file mode 100644 index 002b589dd5..0000000000 --- a/tool/dart_tools/pubspec.yaml +++ /dev/null @@ -1,14 +0,0 @@ -name: dart_tools -description: Dart-based tools for site-www. -publish_to: none - -environment: - sdk: ^3.2.0 - -dependencies: - logging: ^1.2.0 - path: ^1.8.3 - -dev_dependencies: - analysis_defaults: - path: ../../site-shared/packages/analysis_defaults From 7c8c18ab3bd8ee182be986362168861245406dff Mon Sep 17 00:00:00 2001 From: Parker Lougheed Date: Wed, 17 Jan 2024 23:06:22 -0600 Subject: [PATCH 5/5] Remove root script for now --- README.md | 2 +- dart_site.sh | 13 ------------- 2 files changed, 1 insertion(+), 14 deletions(-) delete mode 100755 dart_site.sh diff --git a/README.md b/README.md index ae4e9fb99c..9cf6812271 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ and committed locally, then run the following commands before pushing your work: $ make run # Check/validate example code -$ dart_site check-all +$ dart run dart_site check-all ``` If these scripts report errors or warnings, diff --git a/dart_site.sh b/dart_site.sh deleted file mode 100755 index 23613ee493..0000000000 --- a/dart_site.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -set -e - -REPO_DIR=$(dirname $(dirname $BASH_SOURCE)) - -# Run dart pub get if the packages file doesn't exist yet. -if [[ ! -f "$REPO_DIR.dart_tool/package_config.json" ]]; then - dart pub get -fi - -# Run the dart_site tool and pass all arguments to it. -dart run dart_site "$@"