Fix dev branch CircleCI (#2715)

Summary:
Currently, all `dev` branch `publish_docs` jobs are failing [the Java 11 requirement](https://github.com/facebook/buck/blob/dev/build.xml#L365-L367).

A lot of things need to be changed to pass that requirement:
- Install OpenJDK 11 instead of 8
- Install new Android command-line tools. [The old SDK tools don't support JDK 8](https://stackoverflow.com/a/65782803). Oddly, [the *new* tools don't install to the right location](https://stackoverflow.com/a/67413427), so they need to be moved around a little before they'll work.
- Install Python 3.9.4 (required by Ant)
- Build Buck! Release-branch builds require Java 8, so we can't use those and hope to pass the version check.
- Pass down our local Buck executable to `docs/publish.sh` and `docs/soyweb-prod.sh`. Otherwise, these scripts will try to use the Ant-bootstrapped Java 8 Buck instead of our locally built Java 11 Buck. They'll still use `buck` by default, so local usage is unaffected.

This has been tested by adding a couple tweaks in [a separate branch](https://github.com/egpast/buck/commits/dev-force-circleci) to force `publish_docs` to run on commits and stop it from actually publishing, and then confirming that [CircleCI passes](https://app.circleci.com/pipelines/github/egpast/buck/25/workflows/b049df7e-9e0c-49a8-95d5-eee82aeb40bc/jobs/17).

Pull Request resolved: #2715

Reviewed By: zpao

Pulled By: egpast

fbshipit-source-id: 8163a70188978814c3726c560653079f43fe901a

.circleci/config.yml

@@ -3,14 +3,15 @@ version: 2.1
 orbs:
   win: circleci/[email protected]
 
-install_openjdk8: &install_openjdk8
-  name: Install OpenJDK8
+install_openjdk11: &install_openjdk11
+  name: Install OpenJDK11
   command: |
     if [ "${PLATFORM}" == "linux" ]; then
-      sudo apt-get update && sudo apt-get install openjdk-8-jdk
-      sudo update-java-alternatives -s java-1.8.0-openjdk-amd64
+      sudo apt-get update && sudo apt-get install openjdk-11-jdk
+      sudo update-java-alternatives -s java-1.11.0-openjdk-amd64
     elif [ "${PLATFORM}" == "macos" ]; then
-      brew cask install adoptopenjdk8
+      brew install openjdk@11
+      export PATH="/usr/local/opt/openjdk@11/bin:$PATH"
     fi
     java -version
 
@@ -19,14 +20,15 @@ install_android_sdk: &install_android_sdk
   command: |
     sdk_os="linux"
     if [ "${PLATFORM}" == "macos" ]; then
-      sdk_os="darwin"
+      sdk_os="mac"
     fi
-    sdk_zip_filename="sdk-tools-${sdk_os}-4333796.zip"
-    mkdir -p "${ANDROID_SDK}"
-    cd "${ANDROID_SDK}"
+    sdk_zip_filename="commandlinetools-${sdk_os}-8512546_latest.zip"
+    mkdir -p "${ANDROID_SDK}/cmdline-tools"
+    cd "${ANDROID_SDK}/cmdline-tools"
     curl -O "https://dl.google.com/android/repository/${sdk_zip_filename}"
     unzip "${sdk_zip_filename}"
-    export PATH="${ANDROID_SDK}/tools/bin:${PATH}"
+    mv cmdline-tools latest
+    export PATH="${ANDROID_SDK}/cmdline-tools/latest/bin:${PATH}"
     echo 'y' |sdkmanager --install tools
     echo 'y' |sdkmanager --install platform-tools
     echo 'y' |sdkmanager --install "build-tools;28.0.0"
@@ -40,24 +42,47 @@ install_android_sdk: &install_android_sdk
     fi
 
 install_python: &install_python
-  name: Install Python 3.6.2
+  name: Install Python 3.9.4
   command: |
     if [ "${PLATFORM}" == "macos" ]; then
       brew install pyenv
     fi
-    pyenv install -s 3.6.2
-    pyenv global 3.6.2 system
+    pyenv install -s 3.9.4
+    pyenv global 3.9.4 system
     export PATH="$(pyenv root)/shims:${PATH}"
     python --version
 
+run_ant_build: &run_ant_build
+  name: Run Ant Build
+  command: |
+    if [ "${PLATFORM}" == "macos" ]; then
+      # The latest ant depends on JDK13, install 1.9 instead.
+      brew install [email protected]
+      export PATH="/usr/local/opt/[email protected]/bin:${PATH}"
+    fi
+    cd "${BUCKROOT}"
+    set -ex
+    export ANT_OPTS='-Xmx1000m'
+    ant
+
+run_buck_build: &run_buck_build
+  name: Run Buck Build
+  command: |
+    cd "${BUCKROOT}"
+    echo '-Xmx1024m' > .buckjavaargs.local
+    export PATH="${ANDROID_SDK}/tools/bin:${PATH}"
+    export PATH="$(pyenv root)/shims:${PATH}"
+    set -ex
+    ./bin/buck build buck --out "./${BUCK_PEX_LOCATION}" || { cat "buck-out/log/buck-0.log"; exit 1; }
+
 linux_environment: &linux_environment
   # Use string constant for values, no environment variables
   PLATFORM: "linux"
   BUCKROOT: "/home/circleci/buck"
   ANDROID_SDK: "/home/circleci/android-sdk"
   TERM: "dumb"
   BUCK_NUM_THREADS: 3
-  BUCK_PEX_LOCATION: "./new_buck.pex"
+  BUCK_PEX_LOCATION: "new_buck.pex"
 
 jobs:
   publish_docs:
@@ -69,26 +94,24 @@ jobs:
     steps:
       - checkout
       - run:
-          <<: *install_openjdk8
+          <<: *install_openjdk11
       - run:
           # android_sdk needed to build java docs.
           <<: *install_android_sdk
       - run:
           <<: *install_python
       - run:
-          # We do not want to build buck, install the latest release instead.
-          name: Install Buck
-          command: |
-            url=`curl -sH "Authorization: token ${GITHUB_TOKEN}" https://api.github.com/repos/facebook/buck/releases/latest |grep "browser_download_url.*deb" |awk '{gsub("\"", "", $2); print $2}'`
-            curl -L -O $url
-            filename=`basename ${url}`
-            sudo dpkg -i ${filename} || echo "Warning: Buck installed without dependencies."
+          <<: *run_ant_build
+      - restore_cache:
+          key: v-{{ .Environment.CACHE_VERSION }}-buck-build-{{ .Branch }}
+      - run:
+          <<: *run_buck_build
       - run:
           name: Publish docs
           command: |
             export ANDROID_HOME="${ANDROID_SDK}"
             cd docs
-            ./publish.sh --start-soyweb
+            ./publish.sh --buck-location "${BUCKROOT}/${BUCK_PEX_LOCATION}" --start-soyweb
 
 workflows:
   version: 2.1

docs/publish.sh

@@ -18,7 +18,7 @@ set -e
 
 start_soyweb() {
   echo "Starting soyweb-prod.sh" >&2
-  ./soyweb-prod.sh >&2 &
+  ./soyweb-prod.sh --buck-location "${BUCK_LOCATION}" >&2 &
   local soyweb_pid=$!
   sleep 2
   if ! kill -0 "$soyweb_pid" >/dev/null 2>&1; then
@@ -31,23 +31,29 @@ start_soyweb() {
 
 show_help() {
   cat <<-EOF
-Usage: publish.sh [--start-soyweb] [--keep-files]
-  --start-soyweb Start soyweb and shut it down when the script is finished
-  --keep-files   Keep temporary files after attempting to publish
-  --help Show this help
+Usage: publish.sh [--buck-location BUCK_LOCATION] [--start-soyweb] [--keep-files]
+  --buck-location Sets the location of the Buck executable to use
+  --start-soyweb  Start soyweb and shut it down when the script is finished
+  --keep-files    Keep temporary files after attempting to publish
+  --help          Show this help
 
   Set the environment variables GIT_USER and GITHUB_TOKEN if you want to publish using another
   GitHub account. These variables are already set in CircleCI to automate publishing.
 EOF
   exit 1
 }
 
+BUCK_LOCATION="buck"
 START_SOYWEB=0
 KEEP_FILES=0
 SOYWEB_PID=0
 for arg do
   shift
   case $arg in
+    --buck-location)
+      BUCK_LOCATION="$1"
+      shift
+      ;;
     --start-soyweb) START_SOYWEB=1 ;;
     --keep-files) KEEP_FILES=1 ;;
     --help) show_help ;;
@@ -66,7 +72,7 @@ STATIC_FILES_DIR=$(mktemp -d)
 
 # Always run this from the the docs dir
 cd "$DOCS_DIR"
-buck run //docs:generate_buckconfig_aliases
+"${BUCK_LOCATION}" run //docs:generate_buckconfig_aliases
 
 if ( [[ -n $IS_GIT ]] && ! git diff --quiet ) || hg status | grep -q .; then
   echo "Repository is not clean; refusing to publish"

docs/soyweb-prod.sh

@@ -20,6 +20,28 @@
 # Remove any residual files that could derail build and publication.
 #
 
+show_help() {
+  cat <<-EOF
+Usage: soyweb-prod.sh [--buck-location BUCK_LOCATION]
+  --buck-location Sets the location of the Buck executable to use
+  --help          Show this help
+EOF
+  exit 1
+}
+
+BUCK_LOCATION="buck"
+for arg do
+  shift
+  case $arg in
+    --buck-location)
+      BUCK_LOCATION="$1"
+      shift
+      ;;
+    --help) show_help ;;
+    *) set -- "$@" "$arg" ;;
+  esac
+done
+
 DOCS_DIR=$(dirname "$0")
 BUCK_DIR=$(realpath "$DOCS_DIR/..")
 DOCS_DIR=$(realpath "$DOCS_DIR")
@@ -28,6 +50,6 @@ cd "$BUCK_DIR" || exit
 ant clean
 
 cd "$BUCK_DIR/docs" || exit
-buck run //docs:generate_buckconfig_aliases
+"${BUCK_LOCATION}" run //docs:generate_buckconfig_aliases
 exec java -jar plovr-81ed862.jar soyweb --port 9814 --dir . --globals globals.json