From e9c28914bb5e530ba1633f69ed92c88da815a070 Mon Sep 17 00:00:00 2001 From: klzgrad Date: Fri, 26 Apr 2019 01:40:46 +0800 Subject: [PATCH] Add continuous integration and tests --- .github/workflows/build.yml | 549 ++++++++++++++++++++++++++++++++++++ src/get-android-sys.sh | 19 ++ tests/basic.py | 310 ++++++++++++++++++++ tests/basic.sh | 20 ++ tests/qemu-howto.md | 95 +++++++ 5 files changed, 993 insertions(+) create mode 100644 .github/workflows/build.yml create mode 100755 src/get-android-sys.sh create mode 100644 tests/basic.py create mode 100755 tests/basic.sh create mode 100644 tests/qemu-howto.md diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000..7ff15e6f85 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,549 @@ +name: Build +on: + push: + branches: [master] + paths-ignore: [README.md] + release: + types: [published] +defaults: + run: + shell: bash + working-directory: src +env: + CACHE_EPOCH: 1 + CCACHE_MAXSIZE: 200M + CCACHE_MAXFILES: 0 + SCCACHE_CACHE_SIZE: 200M +jobs: + cache-toolchains-posix: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - name: Cache toolchains (Linux, OpenWrt, Android) + uses: actions/cache@v4 + with: + path: | + src/third_party/llvm-build/Release+Asserts/ + src/gn/ + src/qemu-user-static*.deb + key: toolchains-posix-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache PGO (Linux, OpenWrt) + uses: actions/cache@v4 + with: + path: src/chrome/build/pgo_profiles/ + key: pgo-linux-openwrt-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache AFDO (Android) + uses: actions/cache@v4 + with: + path: src/chrome/android/profiles/ + key: afdo-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache Android NDK (Android) + uses: actions/cache@v4 + with: + path: src/third_party/android_toolchain/ndk/ + key: android-ndk-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - run: ./get-clang.sh + - run: EXTRA_FLAGS='target_os="android"' ./get-clang.sh + - run: | + if [ ! -f qemu-user-static*.deb ]; then + wget https://snapshot.debian.org/archive/debian/20230611T210420Z/pool/main/q/qemu/qemu-user-static_8.0%2Bdfsg-4_amd64.deb + fi + cache-toolchains-win: + runs-on: windows-2022 + steps: + - uses: actions/checkout@v4 + - name: Cache toolchains + uses: actions/cache@v4 + with: + path: | + src/third_party/llvm-build/Release+Asserts/ + src/gn/ + ~/.cargo/bin/ + ~/bin/ninja.exe + key: toolchains-win-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache PGO (win64) + uses: actions/cache@v4 + with: + path: src/chrome/build/pgo_profiles/chrome-win64-* + key: pgo-win64-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache PGO (win32) + uses: actions/cache@v4 + with: + path: src/chrome/build/pgo_profiles/chrome-win32-* + key: pgo-win32-arm64-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - run: EXTRA_FLAGS='target_cpu="x64"' ./get-clang.sh + - run: EXTRA_FLAGS='target_cpu="x86"' ./get-clang.sh + - run: | + if [ ! -f ~/bin/ninja.exe ]; then + curl -LO https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip + unzip ninja-win.zip -d ~/bin + fi + cache-toolchains-mac: + runs-on: macos-13 + steps: + - uses: actions/checkout@v4 + - uses: actions/cache@v4 + with: + path: | + src/third_party/llvm-build/Release+Asserts/ + src/chrome/build/pgo_profiles/chrome-mac-* + src/gn/ + key: toolchains-pgo-mac-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - run: EXTRA_FLAGS='target_cpu="x64"' ./get-clang.sh + - run: EXTRA_FLAGS='target_cpu="arm64"' ./get-clang.sh + linux: + needs: cache-toolchains-posix + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + arch: [x64, x86, arm64, arm, mipsel, mips64el, riscv64] + env: + EXTRA_FLAGS: 'target_cpu="${{ matrix.arch }}"' + BUNDLE: naiveproxy-${{ github.event.release.tag_name }}-${{ github.job }}-${{ matrix.arch }} + steps: + - uses: actions/checkout@v4 + - name: Cache toolchains (Linux, OpenWrt, Android) + uses: actions/cache@v4 + with: + path: | + src/third_party/llvm-build/Release+Asserts/ + src/gn/ + src/qemu-user-static*.deb + key: toolchains-posix-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache PGO (Linux, OpenWrt) + uses: actions/cache@v4 + with: + path: src/chrome/build/pgo_profiles/ + key: pgo-linux-openwrt-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Regenerate Debian keyring + run: | + rm -f ./build/linux/sysroot_scripts/keyring.gpg + GPG_TTY=/dev/null ./build/linux/sysroot_scripts/generate_keyring.sh + - name: Cache sysroot + uses: actions/cache@v4 + with: + path: src/out/sysroot-build/bullseye/bullseye_* + key: sysroot-linux-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - id: ccache-timestamp + run: echo "CCACHE_TIMESTAMP=$(date +%s)" >>$GITHUB_OUTPUT + - name: Cache ccache files + uses: actions/cache@v4 + with: + path: ~/.cache/ccache + key: ccache-linux-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}-${{ steps.ccache-timestamp.outputs.CCACHE_TIMESTAMP }} + restore-keys: ccache-linux-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}- + - name: Install APT packages + run: | + sudo apt update + sudo apt install ninja-build pkg-config ccache bubblewrap + sudo apt remove -y qemu-user-binfmt + sudo dpkg -i qemu-user-static*.deb + # libc6-i386 interferes with x86 build + sudo apt remove libc6-i386 + - run: ./get-clang.sh + - run: ccache -z + - run: ./build.sh + - run: ccache -s + - run: ../tests/basic.sh out/Release/naive + - name: Pack naiveproxy assets + run: | + mkdir ${{ env.BUNDLE }} + cp out/Release/naive config.json ../LICENSE ../USAGE.txt ${{ env.BUNDLE }} + tar cJf ${{ env.BUNDLE }}.tar.xz ${{ env.BUNDLE }} + openssl sha256 out/Release/naive >sha256sum.txt + echo "SHA256SUM=$(cut -d' ' -f2 sha256sum.txt)" >>$GITHUB_ENV + - uses: actions/upload-artifact@v4 + with: + name: ${{ env.BUNDLE }}.tar.xz naive executable sha256 ${{ env.SHA256SUM }} + path: src/sha256sum.txt + - name: Upload naiveproxy assets + if: ${{ github.event_name == 'release' }} + run: gh release upload "${GITHUB_REF##*/}" ${{ env.BUNDLE }}.tar.xz --clobber + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + android: + needs: cache-toolchains-posix + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + include: + - arch: x64 + abi: x86_64 + - arch: x86 + abi: x86 + - arch: arm64 + abi: arm64-v8a + - arch: arm + abi: armeabi-v7a + env: + EXTRA_FLAGS: 'target_cpu="${{ matrix.arch }}" target_os="android"' + BUNDLE: naiveproxy-plugin-${{ github.event.release.tag_name || 'v1.1.1.1-1' }}-${{ matrix.abi }}.apk + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + - name: Cache toolchains (Linux, OpenWrt, Android) + uses: actions/cache@v4 + with: + path: | + src/third_party/llvm-build/Release+Asserts/ + src/gn/ + src/qemu-user-static*.deb + key: toolchains-posix-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache AFDO (Android) + uses: actions/cache@v4 + with: + path: src/chrome/android/profiles/ + key: afdo-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache Android NDK (Android) + uses: actions/cache@v4 + with: + path: src/third_party/android_toolchain/ndk/ + key: android-ndk-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache sysroot + uses: actions/cache@v4 + with: + path: src/out/sysroot-build/android/ + key: sysroot-android-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - id: ccache-timestamp + run: echo "CCACHE_TIMESTAMP=$(date +%s)" >>$GITHUB_OUTPUT + - name: Cache ccache files + uses: actions/cache@v4 + with: + path: ~/.cache/ccache + key: ccache-android-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}-${{ steps.ccache-timestamp.outputs.CCACHE_TIMESTAMP }} + restore-keys: ccache-android-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}- + - name: Install APT packages + run: | + sudo apt update + sudo apt install ninja-build pkg-config ccache bubblewrap + sudo apt remove -y qemu-user-binfmt + sudo dpkg -i qemu-user-static*.deb + # libc6-i386 interferes with x86 build + sudo apt remove libc6-i386 + - run: ./get-clang.sh + - run: ccache -z + - run: ./build.sh + - run: ccache -s + - run: ./get-android-sys.sh + - run: ../tests/basic.sh out/Release/naive + - name: Gradle cache + uses: actions/cache@v4 + with: + path: ~/.gradle + key: gradle-${{ hashFiles('**/*.gradle.kts') }} + - name: Create APK + working-directory: apk + env: + APK_ABI: ${{ matrix.abi }} + APK_VERSION_NAME: ${{ github.event.release.tag_name || 'v1.1.1.1-1' }} + KEYSTORE_PASS: ${{ secrets.KEYSTORE_PASS }} + run: | + mkdir -p app/libs/$APK_ABI + cp ../src/out/Release/naive app/libs/$APK_ABI/libnaive.so + ./gradlew :app:assembleRelease + openssl sha256 app/build/outputs/apk/release/${{ env.BUNDLE }} >sha256sum.txt + echo "SHA256SUM=$(cut -d' ' -f2 sha256sum.txt)" >>$GITHUB_ENV + - uses: actions/upload-artifact@v4 + with: + name: ${{ env.BUNDLE }} sha256 ${{ env.SHA256SUM }} + path: apk/sha256sum.txt + - name: Upload naiveproxy assets + if: ${{ github.event_name == 'release' }} + working-directory: apk/app/build/outputs/apk/release + run: gh release upload "${GITHUB_REF##*/}" ${{ env.BUNDLE }} --clobber + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + win: + needs: cache-toolchains-win + runs-on: windows-2022 + strategy: + fail-fast: false + matrix: + arch: [x64, x86, arm64] + env: + EXTRA_FLAGS: 'target_cpu="${{ matrix.arch }}"' + BUNDLE: naiveproxy-${{ github.event.release.tag_name }}-${{ github.job }}-${{ matrix.arch }} + steps: + - uses: actions/checkout@v4 + - name: Cache toolchains + uses: actions/cache@v4 + with: + path: | + src/third_party/llvm-build/Release+Asserts/ + src/gn/ + ~/.cargo/bin/ + ~/bin/ninja.exe + key: toolchains-win-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache PGO (win64) + if: ${{ matrix.arch == 'x64' }} + uses: actions/cache@v4 + with: + path: src/chrome/build/pgo_profiles/chrome-win64-* + key: pgo-win64-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache PGO (win32) + if: ${{ matrix.arch != 'x64' }} + uses: actions/cache@v4 + with: + path: src/chrome/build/pgo_profiles/chrome-win32-* + key: pgo-win32-arm64-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - id: ccache-timestamp + run: echo "CCACHE_TIMESTAMP=$(date +%s)" >>$GITHUB_OUTPUT + - name: Cache ccache files + uses: actions/cache@v4 + with: + path: ~/AppData/Local/Mozilla/sccache + key: ccache-win-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}-${{ steps.ccache-timestamp.outputs.CCACHE_TIMESTAMP }} + restore-keys: ccache-win-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}- + - run: ./get-clang.sh + - run: ~/.cargo/bin/sccache -z + - run: ./build.sh + - run: ~/.cargo/bin/sccache -s + - run: ../tests/basic.sh out/Release/naive + # No real or emulated environment is available to test this. + if: ${{ matrix.arch != 'arm64' }} + - name: Pack naiveproxy assets + run: | + mkdir ${{ env.BUNDLE }} + cp out/Release/naive config.json ../LICENSE ../USAGE.txt ${{ env.BUNDLE }} + 7z a ${{ env.BUNDLE }}.zip ${{ env.BUNDLE }} + openssl sha256 out/Release/naive.exe >sha256sum.txt + echo "SHA256SUM=$(cut -d' ' -f2 sha256sum.txt)" >>$GITHUB_ENV + - uses: actions/upload-artifact@v4 + with: + name: ${{ env.BUNDLE }}.zip naive executable sha256 ${{ env.SHA256SUM }} + path: src/sha256sum.txt + - name: Upload naiveproxy assets + if: ${{ github.event_name == 'release' }} + run: gh release upload "${GITHUB_REF##*/}" ${{ env.BUNDLE }}.zip --clobber + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + mac: + needs: cache-toolchains-mac + runs-on: macos-13 + strategy: + fail-fast: false + matrix: + arch: [x64, arm64] + env: + EXTRA_FLAGS: 'target_cpu="${{ matrix.arch }}"' + BUNDLE: naiveproxy-${{ github.event.release.tag_name }}-${{ github.job }}-${{ matrix.arch }} + steps: + - uses: actions/checkout@v4 + - name: Cache toolchains and PGO + uses: actions/cache@v4 + with: + path: | + src/third_party/llvm-build/Release+Asserts/ + src/chrome/build/pgo_profiles/chrome-mac-* + src/gn/ + key: toolchains-pgo-mac-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - id: ccache-timestamp + run: echo "CCACHE_TIMESTAMP=$(date +%s)" >>$GITHUB_OUTPUT + - name: Cache ccache files + uses: actions/cache@v4 + with: + path: ~/Library/Caches/ccache + key: ccache-mac-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}-${{ steps.ccache-timestamp.outputs.CCACHE_TIMESTAMP }} + restore-keys: ccache-mac-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}- + - run: brew install ninja ccache + - run: pip install setuptools + - run: ./get-clang.sh + - run: ccache -z + - run: ./build.sh + - run: ccache -s + - run: ../tests/basic.sh out/Release/naive + # No real or emulated environment is available to test this. + if: ${{ matrix.arch != 'arm64' }} + - name: Pack naiveproxy assets + run: | + mkdir ${{ env.BUNDLE }} + cp out/Release/naive config.json ../LICENSE ../USAGE.txt ${{ env.BUNDLE }} + tar cJf ${{ env.BUNDLE }}.tar.xz ${{ env.BUNDLE }} + openssl sha256 out/Release/naive >sha256sum.txt + echo "SHA256SUM=$(cut -d' ' -f2 sha256sum.txt)" >>$GITHUB_ENV + - uses: actions/upload-artifact@v4 + with: + name: ${{ env.BUNDLE }}.tar.xz naive executable sha256 ${{ env.SHA256SUM }} + path: src/sha256sum.txt + - name: Upload naiveproxy assets + if: ${{ github.event_name == 'release' }} + run: gh release upload "${GITHUB_REF##*/}" ${{ env.BUNDLE }}.tar.xz --clobber + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + openwrt: + needs: cache-toolchains-posix + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + include: + - arch: x86_64 + openwrt: "target=x86 subtarget=64" + target_cpu: x64 + - arch: x86 + openwrt: "target=x86 subtarget=generic" + target_cpu: x86 + - arch: aarch64_cortex-a53 + openwrt: "target=sunxi subtarget=cortexa53" + target_cpu: arm64 + extra: 'arm_cpu="cortex-a53"' + - arch: aarch64_cortex-a53-static + openwrt: "target=sunxi subtarget=cortexa53" + target_cpu: arm64 + extra: 'arm_cpu="cortex-a53" build_static=true no_madvise_syscall=true' + - arch: aarch64_cortex-a72 + openwrt: "target=mvebu subtarget=cortexa72" + target_cpu: arm64 + extra: 'arm_cpu="cortex-a72"' + - arch: aarch64_cortex-a72-static + openwrt: "target=mvebu subtarget=cortexa72" + target_cpu: arm64 + extra: 'arm_cpu="cortex-a72" build_static=true no_madvise_syscall=true' + - arch: aarch64_generic + openwrt: "target=rockchip subtarget=armv8" + target_cpu: arm64 + - arch: aarch64_generic-static + openwrt: "target=rockchip subtarget=armv8" + target_cpu: arm64 + extra: "build_static=true no_madvise_syscall=true" + - arch: arm_arm1176jzf-s_vfp + openwrt: "target=bcm27xx subtarget=bcm2708" + target_cpu: arm + extra: 'arm_version=0 arm_cpu="arm1176jzf-s" arm_fpu="vfp" arm_float_abi="hard" arm_use_neon=false arm_use_thumb=false' + - arch: arm_arm926ej-s + openwrt: "target=mxs subtarget=generic" + target_cpu: arm + extra: 'arm_version=0 arm_cpu="arm926ej-s" arm_float_abi="soft" arm_use_neon=false arm_use_thumb=false' + - arch: arm_cortex-a15_neon-vfpv4 + openwrt: "target=armsr subtarget=armv7" + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a15" arm_fpu="neon-vfpv4" arm_float_abi="hard" arm_use_neon=true' + - arch: arm_cortex-a5_vfpv4 + openwrt: "target=at91 subtarget=sama5" + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a5" arm_fpu="vfpv4" arm_float_abi="hard" arm_use_neon=false' + - arch: arm_cortex-a7 + openwrt: "target=mediatek subtarget=mt7629" + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a7" arm_float_abi="soft" arm_use_neon=false' + - arch: arm_cortex-a7_neon-vfpv4 + openwrt: "target=sunxi subtarget=cortexa7" + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a7" arm_fpu="neon-vfpv4" arm_float_abi="hard" arm_use_neon=true' + - arch: arm_cortex-a7_vfpv4 + openwrt: "target=at91 subtarget=sama7" + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a7" arm_fpu="vfpv4" arm_float_abi="hard" arm_use_neon=false' + - arch: arm_cortex-a7_neon-vfpv4-static + openwrt: "target=sunxi subtarget=cortexa7" + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a7" arm_fpu="neon-vfpv4" arm_float_abi="hard" arm_use_neon=true build_static=true no_madvise_syscall=true' + - arch: arm_cortex-a8_vfpv3 + openwrt: "target=sunxi subtarget=cortexa8" + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a8" arm_fpu="vfpv3" arm_float_abi="hard" arm_use_neon=false' + - arch: arm_cortex-a9 + openwrt: "target=bcm53xx subtarget=generic" + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a9" arm_float_abi="soft" arm_use_neon=false' + - arch: arm_cortex-a9-static + openwrt: "target=bcm53xx subtarget=generic" + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a9" arm_float_abi="soft" arm_use_neon=false build_static=true no_madvise_syscall=true' + - arch: arm_cortex-a9_neon + openwrt: "target=zynq subtarget=generic" + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a9" arm_fpu="neon" arm_float_abi="hard" arm_use_neon=true' + - arch: arm_cortex-a9_vfpv3-d16 + openwrt: "target=tegra subtarget=generic" + target_cpu: arm + extra: 'arm_version=0 arm_cpu="cortex-a9" arm_fpu="vfpv3-d16" arm_float_abi="hard" arm_use_neon=false' + - arch: arm_mpcore + openwrt: "target=oxnas subtarget=ox820" + target_cpu: arm + extra: 'arm_version=0 arm_cpu="mpcore" arm_float_abi="soft" arm_use_neon=false arm_use_thumb=false' + - arch: arm_xscale + openwrt: "target=kirkwood subtarget=generic" + target_cpu: arm + extra: 'arm_version=0 arm_cpu="xscale" arm_float_abi="soft" arm_use_neon=false arm_use_thumb=false' + - arch: mipsel_24kc + openwrt: "target=ramips subtarget=rt305x" + target_cpu: mipsel + extra: 'mips_arch_variant="r2" mips_float_abi="soft"' + - arch: mipsel_24kc-static + openwrt: "target=ramips subtarget=rt305x" + target_cpu: mipsel + extra: 'mips_arch_variant="r2" mips_float_abi="soft" build_static=true no_madvise_syscall=true' + - arch: mipsel_mips32 + openwrt: "target=bcm47xx subtarget=generic" + target_cpu: mipsel + extra: 'mips_arch_variant="r1" mips_float_abi="soft"' + - arch: riscv64 + openwrt: "target=sifiveu subtarget=generic" + target_cpu: riscv64 + env: + EXTRA_FLAGS: target_cpu="${{ matrix.target_cpu }}" target_os="openwrt" ${{ matrix.extra }} + OPENWRT_FLAGS: arch=${{ matrix.arch }} release=23.05.0 gcc_ver=12.3.0 ${{ matrix.openwrt }} + BUNDLE: naiveproxy-${{ github.event.release.tag_name }}-${{ github.job }}-${{ matrix.arch }} + steps: + - uses: actions/checkout@v4 + - name: Cache toolchains (Linux, OpenWrt, Android) + uses: actions/cache@v4 + with: + path: | + src/third_party/llvm-build/Release+Asserts/ + src/gn/ + src/qemu-user-static*.deb + key: toolchains-posix-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache PGO (Linux, OpenWrt) + uses: actions/cache@v4 + with: + path: src/chrome/build/pgo_profiles/ + key: pgo-linux-openwrt-${{ hashFiles('CHROMIUM_VERSION') }}-v${{ env.CACHE_EPOCH }} + - name: Cache sysroot + uses: actions/cache@v4 + with: + path: src/out/sysroot-build/openwrt + key: sysroot-openwrt-23.05.0-${{ matrix.arch }}-v${{ env.CACHE_EPOCH }} + - id: ccache-timestamp + run: echo "CCACHE_TIMESTAMP=$(date +%s)" >>$GITHUB_OUTPUT + - name: Cache ccache files + uses: actions/cache@v4 + with: + path: ~/.cache/ccache + key: ccache-openwrt-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}-${{ steps.ccache-timestamp.outputs.CCACHE_TIMESTAMP }} + restore-keys: ccache-openwrt-${{ matrix.arch }}-${{ hashFiles('CHROMIUM_VERSION') }}- + - name: Install APT packages + run: | + sudo apt update + sudo apt install ninja-build pkg-config ccache bubblewrap + sudo apt remove -y qemu-user-binfmt + sudo dpkg -i qemu-user-static*.deb + # libc6-i386 interferes with x86 build + sudo apt remove libc6-i386 + - run: ./get-clang.sh + - run: ccache -z + - run: ./build.sh + - run: ccache -s + - run: ../tests/basic.sh out/Release/naive + - name: Pack naiveproxy assets + run: | + mkdir ${{ env.BUNDLE }} + cp out/Release/naive config.json ../LICENSE ../USAGE.txt ${{ env.BUNDLE }} + tar cJf ${{ env.BUNDLE }}.tar.xz ${{ env.BUNDLE }} + openssl sha256 out/Release/naive >sha256sum.txt + echo "SHA256SUM=$(cut -d' ' -f2 sha256sum.txt)" >>$GITHUB_ENV + - uses: actions/upload-artifact@v4 + with: + name: ${{ env.BUNDLE }}.tar.xz naive executable sha256 ${{ env.SHA256SUM }} + path: src/sha256sum.txt + - name: Upload naiveproxy assets + if: ${{ github.event_name == 'release' }} + run: gh release upload "${GITHUB_REF##*/}" ${{ env.BUNDLE }}.tar.xz --clobber + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/src/get-android-sys.sh b/src/get-android-sys.sh new file mode 100755 index 0000000000..ad9d00d3f2 --- /dev/null +++ b/src/get-android-sys.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -ex + +. ./get-sysroot.sh + +if [ "$WITH_ANDROID_IMG" -a ! -d out/sysroot-build/android/"$WITH_ANDROID_IMG"/system ]; then + curl -O https://dl.google.com/android/repository/sys-img/android/$WITH_ANDROID_IMG.zip + mkdir -p $WITH_ANDROID_IMG/mount + unzip $WITH_ANDROID_IMG.zip '*/system.img' -d $WITH_ANDROID_IMG + # Need mount -t ext4 -o ro,loop,offset=0x100000 for API level of 26+ + sudo mount $WITH_ANDROID_IMG/*/system.img $WITH_ANDROID_IMG/mount + rootfs=out/sysroot-build/android/$WITH_ANDROID_IMG + mkdir -p $rootfs/system/bin $rootfs/system/etc + cp $WITH_ANDROID_IMG/mount/bin/linker* $rootfs/system/bin + cp $WITH_ANDROID_IMG/mount/etc/hosts $rootfs/system/etc + cp -r $WITH_ANDROID_IMG/mount/lib* $rootfs/system + sudo umount $WITH_ANDROID_IMG/mount + rm -rf $WITH_ANDROID_IMG $WITH_ANDROID_IMG.zip +fi diff --git a/tests/basic.py b/tests/basic.py new file mode 100644 index 0000000000..584a7353d6 --- /dev/null +++ b/tests/basic.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +import argparse +import http.server +import os +import sys +import shutil +import ssl +import subprocess +import tempfile +import threading +import time + +parser = argparse.ArgumentParser() +parser.add_argument('--naive', required=True) +parser.add_argument('--rootfs') +parser.add_argument('--target_cpu') +parser.add_argument('--server_protocol', + choices=['http', 'https'], default='https') +argv = parser.parse_args() + +if argv.rootfs: + try: + os.remove(os.path.join(argv.rootfs, 'naive')) + except OSError: + pass + +server_protocol = argv.server_protocol + +_, certfile = tempfile.mkstemp() + +result = subprocess.run( + f'openssl req -new -x509 -keyout {certfile} -out {certfile} -days 1 -nodes -subj /C=XX'.split(), capture_output=True) +result.check_returncode() + +HTTPS_SERVER_HOSTNAME = '127.0.0.1' +HTTP_SERVER_PORT = 60443 if server_protocol == 'https' else 60080 + +httpd = http.server.HTTPServer( + (HTTPS_SERVER_HOSTNAME, HTTP_SERVER_PORT), http.server.SimpleHTTPRequestHandler) +httpd.timeout = 1 +httpd.allow_reuse_address = True +if server_protocol == 'https': + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ssl_context.load_cert_chain(certfile=certfile) + httpd.socket = ssl_context.wrap_socket(httpd.socket, server_side=True) + +httpd_thread = threading.Thread( + target=lambda httpd: httpd.serve_forever(), args=(httpd,), daemon=True) +httpd_thread.start() + + +def test_https_server(hostname, port, proxy=None): + url = f'{server_protocol}://{hostname}:{port}/404' + cmdline = ['curl', '-k', '-s'] + if proxy: + cmdline.extend(['--proxy', proxy]) + cmdline.append(url) + print('subprocess.run', ' '.join(cmdline)) + result = subprocess.run(cmdline, capture_output=True, + timeout=10, text=True, encoding='utf-8') + print(result.stderr, end='') + return 'Error code: 404' in result.stdout + + +assert test_https_server(HTTPS_SERVER_HOSTNAME, + HTTP_SERVER_PORT), 'https server not up' + + +def start_naive(naive_args, config_file): + with_qemu = None + if argv.target_cpu == 'arm64': + with_qemu = 'aarch64' + elif argv.target_cpu == 'arm': + with_qemu = 'arm' + elif argv.target_cpu == 'mipsel': + with_qemu = 'mipsel' + elif argv.target_cpu == 'mips64el': + with_qemu = 'mips64el' + elif argv.target_cpu == 'riscv64': + with_qemu = 'riscv64' + + if argv.rootfs: + if not with_qemu: + if not os.path.exists(os.path.join(argv.rootfs, 'naive')): + shutil.copy2(argv.naive, argv.rootfs) + # bwrap isolates filesystem, config_file needs a copy inside. + if config_file is not None: + shutil.copy2(config_file, argv.rootfs) + cmdline = ['bwrap', '--die-with-parent', '--bind', argv.rootfs, '/', + '--proc', '/proc', '--dev', '/dev', '--chdir', '/', '/naive'] + else: + cmdline = [f'qemu-{with_qemu}-static', + '-L', argv.rootfs, argv.naive] + else: + cmdline = [argv.naive] + cmdline.extend(naive_args) + + proc = subprocess.Popen(cmdline, stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, text=True, encoding='utf-8') + print('subprocess.Popen', ' '.join(cmdline), 'pid:', proc.pid) + + def terminate(proc): + print('proc has timed out') + print('terminate pid', proc.pid) + proc.terminate() + + timeout = threading.Timer(10, terminate, args=(proc,)) + timeout.start() + while True: + if proc.poll() is not None: + timeout.cancel() + return proc.poll() == 0 + + line = proc.stderr.readline().strip() + print(line) + if 'Failed to listen' in line: + timeout.cancel() + print('terminate pid', proc.pid) + proc.terminate() + return 'Failed to listen' + elif 'Listening on ' in line: + timeout.cancel() + return proc + + +port = 10000 + + +def allocate_port_number(): + global port + port += 1 + if port > 60000: + port = 10000 + return port + + +def test_naive_once(proxy, *args, **kwargs): + port_map = {} + + class PortDict(dict): + def __init__(self, port_map): + self._port_map = port_map + + def __getitem__(self, key): + if key.startswith('PORT'): + if key not in self._port_map: + self._port_map[key] = str(allocate_port_number()) + return self._port_map[key] + return key + port_dict = PortDict(port_map) + + proxy = proxy.format_map(port_dict) + + config_file = kwargs.get('config_file') + config_content = kwargs.get('config_content') + if config_content is not None: + config_content = config_content.format_map(port_dict) + print(f"Writing {repr(config_content)} to {config_file}") + with open(config_file, 'w') as f: + f.write('{') + f.write(config_content) + f.write('}') + + naive_procs = [] + + def cleanup(): + if config_file is not None: + os.remove(config_file) + for naive_proc in naive_procs: + print('terminate pid', naive_proc.pid) + naive_proc.terminate() + + for args_instance in args: + naive_args = args_instance.format_map(port_dict).split() + naive_proc = start_naive(naive_args, config_file) + if naive_proc == 'Failed to listen': + cleanup() + return 'Failed to listen' + if not naive_proc: + cleanup() + return False + naive_procs.append(naive_proc) + + result = test_https_server(HTTPS_SERVER_HOSTNAME, HTTP_SERVER_PORT, proxy) + + cleanup() + + return result + + +def test_naive(label, proxy, *args, **kwargs): + RETRIES = 5 + result = None + for i in range(RETRIES): + result = test_naive_once(proxy, *args, **kwargs) + if result == 'Failed to listen': + result = False + print('Retrying...') + time.sleep(1) + continue + break + if result is True: + print('** TEST PASS:', label, end='\n\n') + else: + print('** TEST FAIL:', label, end='\n\n') + sys.exit(1) + + +test_naive('Default config', 'socks5h://127.0.0.1:1080', + '--log') + +test_naive('Default config file', 'socks5h://127.0.0.1:{PORT1}', + '', + config_content='"listen":"socks://127.0.0.1:{PORT1}","log":""', + config_file='config.json') + +test_naive('Custom config file', 'socks5h://127.0.0.1:{PORT1}', + 'custom.json', + config_content='"listen":"socks://127.0.0.1:{PORT1}","log":""', + config_file='custom.json') + +test_naive('Multiple listens - command line', 'socks5h://127.0.0.1:{PORT1}', + '--log --listen=socks://:{PORT1} --listen=http://:{PORT2}') + +test_naive('Multiple listens - command line', 'http://127.0.0.1:{PORT2}', + '--log --listen=socks://:{PORT1} --listen=http://:{PORT2}') + +test_naive('Multiple listens - config file', 'socks5h://127.0.0.1:{PORT1}', + 'multiple-listen.json', + config_content='"listen":["socks://:{PORT1}", "http://:{PORT2}"],"log":""', + config_file='multiple-listen.json') + +test_naive('Multiple listens - config file', 'http://127.0.0.1:{PORT2}', + 'multiple-listen.json', + config_content='"listen":["socks://:{PORT1}", "http://:{PORT2}"],"log":""', + config_file='multiple-listen.json') + +test_naive('Trivial - listen scheme only', 'socks5h://127.0.0.1:1080', + '--log --listen=socks://') + +test_naive('Trivial - listen no host', 'socks5h://127.0.0.1:{PORT1}', + '--log --listen=socks://:{PORT1}') + +test_naive('Trivial - listen no port', 'socks5h://127.0.0.1:1080', + '--log --listen=socks://127.0.0.1') + +test_naive('Trivial - auth', 'socks5h://user:pass@127.0.0.1:{PORT1}', + '--log --listen=socks://user:pass@127.0.0.1:{PORT1}') + +test_naive('Trivial - auth with special chars', 'socks5h://user:^@127.0.0.1:{PORT1}', + '--log --listen=socks://user:^@127.0.0.1:{PORT1}') + +test_naive('Trivial - auth with special chars', 'socks5h://^:^@127.0.0.1:{PORT1}', + '--log --listen=socks://^:^@127.0.0.1:{PORT1}') + +test_naive('Trivial - auth with empty pass', 'socks5h://user:@127.0.0.1:{PORT1}', + '--log --listen=socks://user:@127.0.0.1:{PORT1}') + +test_naive('SOCKS-SOCKS', 'socks5h://127.0.0.1:{PORT1}', + '--log --listen=socks://:{PORT1} --proxy=socks://127.0.0.1:{PORT2}', + '--log --listen=socks://:{PORT2}') + +test_naive('SOCKS-SOCKS - proxy no port', 'socks5h://127.0.0.1:{PORT1}', + '--log --listen=socks://:{PORT1} --proxy=socks://127.0.0.1', + '--log --listen=socks://:1080') + +test_naive('SOCKS-HTTP', 'socks5h://127.0.0.1:{PORT1}', + '--log --listen=socks://:{PORT1} --proxy=http://127.0.0.1:{PORT2}', + '--log --listen=http://:{PORT2}') + +test_naive('HTTP-HTTP', 'http://127.0.0.1:{PORT1}', + '--log --listen=http://:{PORT1} --proxy=http://127.0.0.1:{PORT2}', + '--log --listen=http://:{PORT2}') + +test_naive('HTTP-SOCKS', 'http://127.0.0.1:{PORT1}', + '--log --listen=http://:{PORT1} --proxy=socks://127.0.0.1:{PORT2}', + '--log --listen=socks://:{PORT2}') + +test_naive('SOCKS-SOCKS-SOCKS', 'socks5h://127.0.0.1:{PORT1}', + '--log --listen=socks://:{PORT1} --proxy=socks://127.0.0.1:{PORT2}', + '--log --listen=socks://:{PORT2} --proxy=socks://127.0.0.1:{PORT3}', + '--log --listen=socks://:{PORT3}') + +test_naive('SOCKS-HTTP-SOCKS', 'socks5h://127.0.0.1:{PORT1}', + '--log --listen=socks://:{PORT1} --proxy=http://127.0.0.1:{PORT2}', + '--log --listen=http://:{PORT2} --proxy=socks://127.0.0.1:{PORT3}', + '--log --listen=socks://:{PORT3}') + +test_naive('HTTP-SOCKS-HTTP', 'http://127.0.0.1:{PORT1}', + '--log --listen=http://:{PORT1} --proxy=socks://127.0.0.1:{PORT2}', + '--log --listen=socks://:{PORT2} --proxy=http://127.0.0.1:{PORT3}', + '--log --listen=http://:{PORT3}') + +test_naive('HTTP-HTTP-HTTP', 'http://127.0.0.1:{PORT1}', + '--log --listen=http://:{PORT1} --proxy=http://127.0.0.1:{PORT2}', + '--log --listen=http://:{PORT2} --proxy=http://127.0.0.1:{PORT3}', + '--log --listen=http://:{PORT3}') + +test_naive('HTTP-HTTP (with auth)', 'http://127.0.0.1:{PORT1}', + '--log --listen=http://:{PORT1} --proxy=http://hello:world@127.0.0.1:{PORT2}', + '--log --listen=http://hello:world@127.0.0.1:{PORT2}') + +test_naive('HTTPa-HTTPb,HTTPc (chaining with remote loop)', 'http://127.0.0.1:{PORT1}', + '--log --listen=http://:{PORT2}', + '--log --listen=http://:{PORT1} --proxy=http://127.0.0.1:{PORT2},http://127.0.0.1:{PORT2}') + +test_naive('HTTPa-HTTPb,HTTPc (chaining with multiple auth)', 'http://127.0.0.1:{PORT1}', + '--log --listen=http://hello:world2@127.0.0.1:{PORT2}', + '--log --listen=http://hello:world3@127.0.0.1:{PORT3}', + '--log --listen=http://127.0.0.1:{PORT1} --proxy=http://hello:world2@127.0.0.1:{PORT2},http://hello:world3@127.0.0.1:{PORT3}') diff --git a/tests/basic.sh b/tests/basic.sh new file mode 100755 index 0000000000..70b920f019 --- /dev/null +++ b/tests/basic.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +set -ex + +script_dir=$(dirname "$PWD/$0") + +[ "$1" ] || exit 1 +naive="$PWD/$1" + +. ./get-sysroot.sh + +if [ "$WITH_ANDROID_IMG" ]; then + rootfs="$PWD/out/sysroot-build/android/$WITH_ANDROID_IMG" +elif [ "$WITH_SYSROOT" ]; then + rootfs="$PWD/$WITH_SYSROOT" +fi + +cd /tmp +python3 "$script_dir"/basic.py --naive="$naive" --rootfs="$rootfs" --target_cpu="$target_cpu" --server_protocol=https +python3 "$script_dir"/basic.py --naive="$naive" --rootfs="$rootfs" --target_cpu="$target_cpu" --server_protocol=http diff --git a/tests/qemu-howto.md b/tests/qemu-howto.md new file mode 100644 index 0000000000..c83a22114c --- /dev/null +++ b/tests/qemu-howto.md @@ -0,0 +1,95 @@ +# Debug ARM Cortex-A9 static in QEMU + +``` +export EXTRA_FLAGS='target_cpu="arm" target_os="openwrt" arm_version=0 arm_cpu="cortex-a9" arm_float_abi="soft" arm_use_neon=false build_static=true no_madvise_syscall=true' +export OPENWRT_FLAGS='arch=arm_cortex-a9-static release=23.05.0 gcc_ver=12.3.0 target=bcm53xx subtarget=generic' +./get-clang.sh +./build.sh +``` + +See https://wiki.qemu.org/Documentation/Networking for example. + +``` +$ wget https://downloads.openwrt.org/releases/23.05.2/targets/armsr/armv7/openwrt-23.05.2-armsr-armv7-generic-initramfs-kernel.bin + +$ qemu-system-arm -nographic -M virt -m 1024 -kernel openwrt-23.05.2-armsr-armv7-generic-initramfs-kernel.bin -device virtio-net,netdev=net0 -netdev user,id=net0,hostfwd=tcp::5555-:1080 +... + +root@OpenWrt:/# ip link del br-lan +root@OpenWrt:/# ip addr add 10.0.2.15/24 dev eth0 +root@OpenWrt:/# ip route add default via 10.0.2.2 +root@OpenWrt:/# nft flush ruleset +root@OpenWrt:/# echo nameserver 10.0.2.3 >/etc/resolv.conf +root@OpenWrt:/# scp user@10.0.2.2:/tmp/naive . +root@OpenWrt:/# ./naive --listen=socks://0.0.0.0:1080 --proxy=https://user:pass@example.com --log + +user@host:/tmp$ curl -v --proxy socks5h://127.0.0.1:5555 example.com +``` + +Install GDB +``` +root@OpenWrt:/# sed -i -e "s/https/http/" /etc/opkg/distfeeds.conf +root@OpenWrt:/# echo option http_proxy http://10.0.2.2:8080/ >>/etc/opkg.conf +root@OpenWrt:/# opkg update +root@OpenWrt:/# opkg install gdb +``` + +# Debug ARM64 static in QEMU + +``` +export EXTRA_FLAGS='target_cpu="arm64" target_os="openwrt" arm_cpu="cortex-a53" build_static=true no_madvise_syscall=true' +export OPENWRT_FLAGS='arch=aarch64_cortex-a53-static release=23.05.0 gcc_ver=12.3.0 target=sunxi subtarget=cortexa53' +./get-clang.sh +./build.sh +``` + +``` +$ wget https://downloads.openwrt.org/releases/23.05.2/targets/armsr/armv8/openwrt-23.05.2-armsr-armv8-generic-initramfs-kernel.bin + +$ qemu-system-aarch64 -m 1024 -M virt -cpu cortex-a53 -nographic -kernel openwrt-23.05.2-armsr-armv8-generic-initramfs-kernel.bin -device virtio-net,netdev=net0 -netdev user,id=net0,hostfwd=tcp::5555-:1080 +... + +root@OpenWrt:/# ip link del br-lan +root@OpenWrt:/# ip addr add 10.0.2.15/24 dev eth0 +root@OpenWrt:/# ip route add default via 10.0.2.2 +root@OpenWrt:/# nft flush ruleset +root@OpenWrt:/# echo nameserver 10.0.2.3 >/etc/resolv.conf +root@OpenWrt:/# scp user@10.0.2.2:/tmp/naive . +root@OpenWrt:/# ./naive --listen=socks://0.0.0.0:1080 --proxy=https://user:pass@example.com --log +user@host:/tmp$ curl -v --proxy socks5h://127.0.0.1:5555 example.com +``` + +# Debug MIPSEL static in QEMU + +``` +export EXTRA_FLAGS='target_cpu="mipsel" target_os="openwrt" mips_arch_variant="r2" mips_float_abi="soft" build_static=true no_madvise_syscall=true' +export OPENWRT_FLAGS='arch=mipsel_24kc-static release=23.05.0 gcc_ver=12.3.0 target=ramips subtarget=rt305x' +./get-clang.sh +./build.sh +``` + +``` +$ wget https://downloads.openwrt.org/snapshots/targets/malta/le/lede-malta-le-vmlinux-initramfs.elf + +$ qemu-system-mipsel -nographic -M malta -kernel lede-malta-le-vmlinux-initramfs.elf -m 64 -device virtio-net,netdev=net0 -netdev user,id=net0,hostfwd=tcp::5555-:1080 +... +(eth0 is set up by DHCP) + +root@LEDE:/# iptables -F +(scp is too old) +user@host:/tmp$ nc -l -p 2222 <./naive +root@LEDE:/# nc 10.0.2.2 2222 >naive +^C +root@LEDE:/# chmod +x ./naive +user@host:/tmp$ nc -l -p2222 /etc/ssl/certs/ca-certificates.crt +^C +root@LEDE:/# ./naive --listen=socks://0.0.0.0:1080 --proxy=https://user:pass@example.com --log +user@host:/tmp$ curl -v --proxy socks5h://127.0.0.1:5555 example.com +``` + +## To exit QEMU in -nographic: + +Press Ctrl-A +Press X