From 52e4c59dfbc27efe00c9ae98672d9ce5e21f4302 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Thu, 16 Jun 2022 12:53:47 -0700 Subject: [PATCH 01/29] Trim dependencies and test more distros --- .github/workflows/setup-sh.yml | 51 +++++++++++++++++++++++---- setup.sh | 63 ++++++---------------------------- 2 files changed, 55 insertions(+), 59 deletions(-) diff --git a/.github/workflows/setup-sh.yml b/.github/workflows/setup-sh.yml index ea4fa11e8..01261f20e 100644 --- a/.github/workflows/setup-sh.yml +++ b/.github/workflows/setup-sh.yml @@ -12,13 +12,11 @@ on: - ".github/workflows/setup-sh.yml" jobs: - build: + build_windows: strategy: matrix: os: - - ubuntu-20.04 - - ubuntu-22.04 - - macos-12 + - windows-2022 python: - "3.8" - "3.9" @@ -28,18 +26,59 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python }} + - name: Run setup.ps1 + if: matrix.os == 'windows-2022' + run: .\setup.ps1 + - name: Test angr import + run: python -c "import angr; print('angr imports!')" + build_macos: + strategy: + matrix: + os: + - macos-12 + python: + - "3.8" + - "3.9" + - "3.10" + fail-fast: false + name: Run setup on ${{ matrix.os }} with python ${{ matrix.python }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 - name: Upgrade bash if: matrix.os == 'macos-12' run: brew install bash - - uses: actions/setup-python@v3 with: python-version: ${{ matrix.python }} - - name: Run setup.sh run: ./setup.sh -i -u + - name: Test angr import + run: python -c "import angr; print('angr imports!')" + build_linux: + strategy: + matrix: + container: + - ubuntu:bionic + - ubuntu:focal + - archlinux:latest + - fedora:36 + - almalinux:8 + - almalinux:9 + - opensuse/leap:15 + - opensuse/tumbleweed:latest + fail-fast: false + name: Run setup on ${{ container.os }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Run setup.sh + run: ./setup.sh -i -u - name: Test angr import run: python -c "import angr; print('angr imports!')" diff --git a/setup.sh b/setup.sh index 301306729..413db7ffc 100755 --- a/setup.sh +++ b/setup.sh @@ -45,16 +45,20 @@ then fi -DEBS=${DEBS-python3-pip python3-dev python3-setuptools build-essential libxml2-dev libxslt1-dev git libffi-dev cmake libreadline-dev libtool debootstrap debian-archive-keyring libglib2.0-dev libpixman-1-dev qtdeclarative5-dev binutils-multiarch nasm libssl-dev} +# macOS HOMEBREW_DEBS=${HOMEBREW_DEBS-python3 libxml2 libxslt libffi cmake libtool glib binutils nasm patchelf} -ARCHDEBS=${ARCHDEBS-python-pip libxml2 libxslt git libffi cmake readline libtool debootstrap glib2 pixman qt5-base binutils nasm} -ARCHCOMDEBS=${ARCHCOMDEBS} -RPMS=${RPMS-gcc gcc-c++ make python3-pip python3-devel python3-setuptools libxml2-devel libxslt-devel git libffi-devel cmake readline-devel libtool debootstrap debian-keyring glib2-devel pixman-devel qt5-qtdeclarative-devel binutils-x86_64-linux-gnu nasm openssl-devel} -OPENSUSE_RPMS=${OPENSUSE_RPMS-gcc gcc-c++ make python3-pip python3-devel python3-setuptools libxml2-devel libxslt-devel git libffi-devel cmake readline-devel libtool debootstrap glib2-devel libpixman-1-0-devel libQt5Core5 libqt5-qtdeclarative-devel binutils nasm libopenssl-devel} + +# Linux distros +DEBS=${DEBS-python3-dev python3-venv} +ARCHDEBS=${ARCHDEBS-python base-devel} +RPMS=${RPMS-python3-devel} +OPENSUSE_RPMS=${OPENSUSE_RPMS-python3-devel} + REPOS=${REPOS-archinfo pyvex cle claripy ailment angr angr-doc binaries} REPOS_CPYTHON=${REPOS_CPYTHON-angr-management} # archr is Linux only because of shellphish-qemu dependency if [ `uname` == "Linux" ]; then REPOS="${REPOS} archr"; fi + declare -A EXTRA_DEPS EXTRA_DEPS["angr"]="sqlalchemy unicorn==2.0.1.post1" EXTRA_DEPS["pyvex"]="--pre capstone" @@ -156,6 +160,7 @@ function error if [ "$INSTALL_REQS" -eq 1 ] then + info "Installing dependencies..." if [ $EUID -eq 0 ] then export SUDO= @@ -164,32 +169,15 @@ then fi if [ -e /etc/debian_version ] then - if ! (dpkg --print-foreign-architectures | grep -q i386) - then - info "Adding i386 architectures..." - $SUDO dpkg --add-architecture i386 - $SUDO apt-get update - fi - info "Installing dependencies..." $SUDO apt-get install -y $DEBS elif [ -e /etc/pacman.conf ] then - if ! grep --quiet "^\[multilib\]" /etc/pacman.conf; - then - info "Adding i386 architectures..." - $SUDO sed 's/^\(#\[multilib\]\)/\[multilib\]/' /tmp/pacman.conf - $SUDO sed '/^\[multilib\]/{n;s/^#//}' /etc/pacman.conf - $SUDO pacman -Syu - fi - info "Installing dependencies..." $SUDO pacman -S --noconfirm --needed $ARCHDEBS elif [ -e /etc/fedora-release ] then - info "Installing dependencies..." $SUDO dnf install -y $RPMS elif [ -e /etc/zypp ] then - info "Installing dependencies..." $SUDO zypper install -y $OPENSUSE_RPMS elif [ $IS_MACOS -eq 1 ] then @@ -206,37 +194,6 @@ then fi fi -info "Checking dependencies..." -if [ -e /etc/debian_version ] -then - INSTALLED_DEBS=$(dpkg --get-selections $DEBS 2>/dev/null) - MISSING_DEBS="" - for REQ in $DEBS; do - if ! grep "$REQ" <<<$INSTALLED_DEBS >/dev/null 2>/dev/null; then - MISSING_DEBS="$REQ $MISSING_DEBS" - fi - done - [ -n "$MISSING_DEBS" ] && error "Please install the following packages: $MISSING_DEBS" -elif [ -e /etc/pacman.conf ] -then - [ $(pacman -Qi $ARCHDEBS 2>&1 | grep "was not found" | wc -l) -ne 0 ] && error "Please install the following packages: $ARCHDEBS" - [ $(pacman -Qi $ARCHCOMDEBS 2>&1 | grep "was not found" | wc -l) -ne 0 ] && error "Please install the following packages from AUR (yaourt -S )): $ARCHCOMDEBS" -elif [ -e /etc/fedora-release ] -then - [ $(rpm -q $RPMS 2>&1 | grep "is not installed" | wc -l) -ne 0 ] && error "Please install the following packages: $RPMS" -elif [ -e /etc/zypp ] -then - [ $(rpm -q $OPENSUSE_RPMS 2>&1 | grep "is not installed" | wc -l) -ne 0 ] && error "Please install the following packages: $OPENSUSE_RPMS" -elif [ -e /etc/NIXOS ] -then - [ -z "$IN_NIX_SHELL" ] && error "Please run in the provided shell.nix" -elif [ $IS_MACOS -eq 1 ] -then - [ $(brew ls --versions $HOMEBREW_DEBS | wc -l) -ne $(echo $HOMEBREW_DEBS | wc -w) ] && error "Please install the following packages from homebrew: $HOMEBREW_DEBS" -else - warning "WARNING: make sure you have dependencies installed.\nThe debian equivalents are: $DEBS." -fi - if [ -n "$ANGR_VENV" ] then info "Enabling virtualenvwrapper." From 8531ee0c20e499f27a4f4cf12822097138577eac Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Thu, 16 Jun 2022 12:56:03 -0700 Subject: [PATCH 02/29] Also trim homebrew deps --- setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.sh b/setup.sh index 413db7ffc..82fe9fbc4 100755 --- a/setup.sh +++ b/setup.sh @@ -46,7 +46,7 @@ fi # macOS -HOMEBREW_DEBS=${HOMEBREW_DEBS-python3 libxml2 libxslt libffi cmake libtool glib binutils nasm patchelf} +HOMEBREW_DEBS=${HOMEBREW_DEBS-python3} # Linux distros DEBS=${DEBS-python3-dev python3-venv} From f81735a294a4a27b12f6357bb6f060a4692f974a Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Thu, 16 Jun 2022 13:03:53 -0700 Subject: [PATCH 03/29] Fix container definition for CI --- .github/workflows/setup-sh.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/setup-sh.yml b/.github/workflows/setup-sh.yml index 01261f20e..514f216be 100644 --- a/.github/workflows/setup-sh.yml +++ b/.github/workflows/setup-sh.yml @@ -73,8 +73,10 @@ jobs: - opensuse/leap:15 - opensuse/tumbleweed:latest fail-fast: false - name: Run setup on ${{ container.os }} + name: Run setup on ${{ container }} runs-on: ubuntu-latest + container: + image: ${{ container }} steps: - uses: actions/checkout@v3 - name: Run setup.sh From 0c968fdc1355779782c79c437156fe0bf3c0c1da Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Thu, 16 Jun 2022 13:18:21 -0700 Subject: [PATCH 04/29] Fix matrix usage --- .github/workflows/setup-sh.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/setup-sh.yml b/.github/workflows/setup-sh.yml index 514f216be..facce11bc 100644 --- a/.github/workflows/setup-sh.yml +++ b/.github/workflows/setup-sh.yml @@ -73,10 +73,10 @@ jobs: - opensuse/leap:15 - opensuse/tumbleweed:latest fail-fast: false - name: Run setup on ${{ container }} + name: Run setup on ${{ matrix.container }} runs-on: ubuntu-latest container: - image: ${{ container }} + image: ${{ matrix.container }} steps: - uses: actions/checkout@v3 - name: Run setup.sh From 88b495a5db68cefdedd66dff5d5a252bad6d92cf Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Thu, 16 Jun 2022 13:34:24 -0700 Subject: [PATCH 05/29] Fix depedendency installation issues --- setup.sh | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/setup.sh b/setup.sh index 82fe9fbc4..886ebb4a0 100755 --- a/setup.sh +++ b/setup.sh @@ -46,13 +46,13 @@ fi # macOS -HOMEBREW_DEBS=${HOMEBREW_DEBS-python3} +HOMEBREW_DEBS=${HOMEBREW_DEBS-git python3} # Linux distros -DEBS=${DEBS-python3-dev python3-venv} -ARCHDEBS=${ARCHDEBS-python base-devel} -RPMS=${RPMS-python3-devel} -OPENSUSE_RPMS=${OPENSUSE_RPMS-python3-devel} +DEBS=${DEBS-git python3-dev python3-venv} +ARCHDEBS=${ARCHDEBS-git python base-devel} +RPMS=${RPMS-git python3-devel} +OPENSUSE_RPMS=${OPENSUSE_RPMS-git python3-devel} REPOS=${REPOS-archinfo pyvex cle claripy ailment angr angr-doc binaries} REPOS_CPYTHON=${REPOS_CPYTHON-angr-management} @@ -167,27 +167,22 @@ then else export SUDO=sudo fi - if [ -e /etc/debian_version ] - then + source /etc/os-release + if [ "$ID_LIKE" -eq "*debian*" ]; then + $SUDO apt-get update $SUDO apt-get install -y $DEBS - elif [ -e /etc/pacman.conf ] - then - $SUDO pacman -S --noconfirm --needed $ARCHDEBS - elif [ -e /etc/fedora-release ] - then - $SUDO dnf install -y $RPMS - elif [ -e /etc/zypp ] - then + if [ "$ID_LIKE" -eq "*fedora*" ]; then + $SUDO dnf install -yq $RPMS + elif [ "$ID_LIKE" -eq "*suse*" ]; then $SUDO zypper install -y $OPENSUSE_RPMS - elif [ $IS_MACOS -eq 1 ] - then - if ! which brew > /dev/null; - then + elif [ "$ID" -eq "arch" ]; then + $SUDO pacman -Sy --noconfirm --needed $ARCHDEBS + elif [ $IS_MACOS -eq 1 ]; then + if ! which brew > /dev/null; then error "Your system doesn't have homebrew installed, I don't know how to install the dependencies.\nPlease install homebrew: https://brew.sh/\nOr install the equivalent of these homebrew packages: $HOMEBREW_DEBS." fi brew install $HOMEBREW_DEBS - elif [ -e /etc/NIXOS ] - then + elif [ -e /etc/NIXOS ]; then info "Doing nothing about dependencies installation for NixOS, as they are provided via shell.nix..." else error "We don't know which dependencies to install for this sytem.\nPlease install the equivalents of these debian packages: $DEBS." From 22cfc5a13d32a1a6062215978199e89daa2f22e3 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Thu, 16 Jun 2022 13:36:45 -0700 Subject: [PATCH 06/29] Fix typo --- setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.sh b/setup.sh index 886ebb4a0..74b415e5d 100755 --- a/setup.sh +++ b/setup.sh @@ -171,7 +171,7 @@ then if [ "$ID_LIKE" -eq "*debian*" ]; then $SUDO apt-get update $SUDO apt-get install -y $DEBS - if [ "$ID_LIKE" -eq "*fedora*" ]; then + elif [ "$ID_LIKE" -eq "*fedora*" ]; then $SUDO dnf install -yq $RPMS elif [ "$ID_LIKE" -eq "*suse*" ]; then $SUDO zypper install -y $OPENSUSE_RPMS From bffac832ac6618371570cddd96bdfb51fe109c42 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Thu, 16 Jun 2022 16:19:05 -0700 Subject: [PATCH 07/29] Try to figure out how [ works --- setup.sh | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/setup.sh b/setup.sh index 74b415e5d..1220ae09d 100755 --- a/setup.sh +++ b/setup.sh @@ -167,25 +167,29 @@ then else export SUDO=sudo fi - source /etc/os-release - if [ "$ID_LIKE" -eq "*debian*" ]; then - $SUDO apt-get update - $SUDO apt-get install -y $DEBS - elif [ "$ID_LIKE" -eq "*fedora*" ]; then - $SUDO dnf install -yq $RPMS - elif [ "$ID_LIKE" -eq "*suse*" ]; then - $SUDO zypper install -y $OPENSUSE_RPMS - elif [ "$ID" -eq "arch" ]; then - $SUDO pacman -Sy --noconfirm --needed $ARCHDEBS - elif [ $IS_MACOS -eq 1 ]; then + if [ $IS_MACOS -eq 1 ]; then if ! which brew > /dev/null; then error "Your system doesn't have homebrew installed, I don't know how to install the dependencies.\nPlease install homebrew: https://brew.sh/\nOr install the equivalent of these homebrew packages: $HOMEBREW_DEBS." fi brew install $HOMEBREW_DEBS elif [ -e /etc/NIXOS ]; then info "Doing nothing about dependencies installation for NixOS, as they are provided via shell.nix..." + elif [ -f /etc/os-release ]; do + source /etc/os-release + if [ "$ID_LIKE" = "*debian*" ]; then + $SUDO apt-get update + $SUDO apt-get install -y $DEBS + elif [ "$ID_LIKE" = "*fedora*" ]; then + $SUDO dnf install -yq $RPMS + elif [ "$ID_LIKE" = "*suse*" ]; then + $SUDO zypper install -y $OPENSUSE_RPMS + elif [ "$ID" = "arch" ]; then + $SUDO pacman -Sy --noconfirm --needed $ARCHDEBS + else + error "We don't recognize this system. Please install equivelents of these debian packages: $DEBS" + fi else - error "We don't know which dependencies to install for this sytem.\nPlease install the equivalents of these debian packages: $DEBS." + error "We don't recognize this system. Please install the equivalents of these debian packages: $DEBS." fi fi From 9019666b425f92430de945fd5c749361df21019e Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Thu, 16 Jun 2022 17:53:06 -0700 Subject: [PATCH 08/29] Fix another typo --- setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.sh b/setup.sh index 1220ae09d..13dbe7365 100755 --- a/setup.sh +++ b/setup.sh @@ -174,7 +174,7 @@ then brew install $HOMEBREW_DEBS elif [ -e /etc/NIXOS ]; then info "Doing nothing about dependencies installation for NixOS, as they are provided via shell.nix..." - elif [ -f /etc/os-release ]; do + elif [ -f /etc/os-release ]; then source /etc/os-release if [ "$ID_LIKE" = "*debian*" ]; then $SUDO apt-get update From fcceed5b9427d49577f1b960fc748eb9960a476c Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Tue, 5 Jul 2022 16:44:55 -0700 Subject: [PATCH 09/29] Improve distro detection --- setup.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.sh b/setup.sh index 13dbe7365..1a3e9e5e3 100755 --- a/setup.sh +++ b/setup.sh @@ -176,12 +176,12 @@ then info "Doing nothing about dependencies installation for NixOS, as they are provided via shell.nix..." elif [ -f /etc/os-release ]; then source /etc/os-release - if [ "$ID_LIKE" = "*debian*" ]; then + if [[ "$ID_LIKE" =~ "debian" ]]; then $SUDO apt-get update $SUDO apt-get install -y $DEBS - elif [ "$ID_LIKE" = "*fedora*" ]; then + elif [[ "$ID_LIKE" =~ "fedora" ]]; then $SUDO dnf install -yq $RPMS - elif [ "$ID_LIKE" = "*suse*" ]; then + elif [[ "$ID_LIKE" =~ "suse" ]]; then $SUDO zypper install -y $OPENSUSE_RPMS elif [ "$ID" = "arch" ]; then $SUDO pacman -Sy --noconfirm --needed $ARCHDEBS From 48c93592d22ae55483b3d4955a2b6300224ee019 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Tue, 5 Jul 2022 16:53:53 -0700 Subject: [PATCH 10/29] Specify python3 when checking python platform --- setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.sh b/setup.sh index 1a3e9e5e3..5f5e50678 100755 --- a/setup.sh +++ b/setup.sh @@ -265,7 +265,7 @@ then fi # Must happen after virutalenv is enabled to correctly detect python implementation -implementation=$(python -c "import sys; print(sys.implementation.name)") +implementation=$(python3 -c "import sys; print(sys.implementation.name)") if [ "$implementation" == "cpython" ]; then REPOS="${REPOS} $REPOS_CPYTHON"; fi # Install build dependencies until build isolation can be enabled From 5ccd1b2ee6c5f7e049b3cdfd213dca1d09f7a534 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Tue, 5 Jul 2022 17:04:06 -0700 Subject: [PATCH 11/29] Check ID and ID_LIKE --- setup.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup.sh b/setup.sh index 5f5e50678..d6324e7b9 100755 --- a/setup.sh +++ b/setup.sh @@ -176,15 +176,15 @@ then info "Doing nothing about dependencies installation for NixOS, as they are provided via shell.nix..." elif [ -f /etc/os-release ]; then source /etc/os-release - if [[ "$ID_LIKE" =~ "debian" ]]; then + if [[ "$ID $ID_LIKE" =~ "debian" ]]; then $SUDO apt-get update - $SUDO apt-get install -y $DEBS - elif [[ "$ID_LIKE" =~ "fedora" ]]; then + $SUDO apt-get install -yq $DEBS + elif [[ "$ID $ID_LIKE" =~ "fedora" ]]; then $SUDO dnf install -yq $RPMS - elif [[ "$ID_LIKE" =~ "suse" ]]; then + elif [[ "$ID $ID_LIKE" =~ "suse" ]]; then $SUDO zypper install -y $OPENSUSE_RPMS - elif [ "$ID" = "arch" ]; then - $SUDO pacman -Sy --noconfirm --needed $ARCHDEBS + elif [ "$ID $ID_LIKE" ~= "arch" ]; then + $SUDO pacman -Syq --noconfirm --needed $ARCHDEBS else error "We don't recognize this system. Please install equivelents of these debian packages: $DEBS" fi From 7a1bd0e966c1290ea58c10683adb6cb498c489cb Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Wed, 6 Jul 2022 11:46:28 -0700 Subject: [PATCH 12/29] Fix dectection of arch linux --- setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.sh b/setup.sh index d6324e7b9..62ad56c60 100755 --- a/setup.sh +++ b/setup.sh @@ -183,7 +183,7 @@ then $SUDO dnf install -yq $RPMS elif [[ "$ID $ID_LIKE" =~ "suse" ]]; then $SUDO zypper install -y $OPENSUSE_RPMS - elif [ "$ID $ID_LIKE" ~= "arch" ]; then + elif [ "$ID $ID_LIKE" =~ "arch" ]; then $SUDO pacman -Syq --noconfirm --needed $ARCHDEBS else error "We don't recognize this system. Please install equivelents of these debian packages: $DEBS" From 6c81c7a0ca0107ee8f4437cb0094253b556214dc Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Wed, 6 Jul 2022 11:48:48 -0700 Subject: [PATCH 13/29] Disable opensuse until tar issue can be sorted --- .github/workflows/setup-sh.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/setup-sh.yml b/.github/workflows/setup-sh.yml index facce11bc..f416f0389 100644 --- a/.github/workflows/setup-sh.yml +++ b/.github/workflows/setup-sh.yml @@ -70,8 +70,9 @@ jobs: - fedora:36 - almalinux:8 - almalinux:9 - - opensuse/leap:15 - - opensuse/tumbleweed:latest + # OpenSUSE is disabled because they don't include tar + # - opensuse/leap:15 + # - opensuse/tumbleweed:latest fail-fast: false name: Run setup on ${{ matrix.container }} runs-on: ubuntu-latest From ceb5edebe1b427cd9a2d4ae765200496e4482e22 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Wed, 6 Jul 2022 11:58:14 -0700 Subject: [PATCH 14/29] Use pip_install function to install dependencies --- setup.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.sh b/setup.sh index 62ad56c60..87d6b3148 100755 --- a/setup.sh +++ b/setup.sh @@ -323,6 +323,9 @@ function clone_repo function pip_install { + # Use --user flag if not using a venv + if [ -n "$VIRTUAL_ENV" ]; then USER_FLAG="--user"; else USER_FLAG=""; fi + debug "pip-installing: $@." if ! pip3 install $PIP_OPTIONS $@ then @@ -405,7 +408,7 @@ then for PACKAGE in $TO_INSTALL; do info "Installing $PACKAGE." [ -n "${EXTRA_DEPS[$PACKAGE]}" ] && pip_install ${EXTRA_DEPS[$PACKAGE]} - pip_install --no-build-isolation -e $PACKAGE + pip_install --no-build-isolation -e ./$PACKAGE done info "Installing some other helpful stuff" From f534d558450dbe342e2e943998de2a8ed527f960 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Wed, 6 Jul 2022 12:02:30 -0700 Subject: [PATCH 15/29] Add -v --- .github/workflows/setup-sh.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/setup-sh.yml b/.github/workflows/setup-sh.yml index f416f0389..2f150b003 100644 --- a/.github/workflows/setup-sh.yml +++ b/.github/workflows/setup-sh.yml @@ -56,7 +56,7 @@ jobs: with: python-version: ${{ matrix.python }} - name: Run setup.sh - run: ./setup.sh -i -u + run: ./setup.sh -i -u -v - name: Test angr import run: python -c "import angr; print('angr imports!')" From 14c2d6840072ef8b0cef933b683ae44539535756 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Wed, 6 Jul 2022 17:06:13 -0700 Subject: [PATCH 16/29] Move clone and install functions to top of script --- setup.sh | 124 +++++++++++++++++++++++++++---------------------------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/setup.sh b/setup.sh index 87d6b3148..d57f4146a 100755 --- a/setup.sh +++ b/setup.sh @@ -158,6 +158,68 @@ function error exit 1 } +function try_remote +{ + URL=$1 + debug "Trying to clone from $URL" + rm -f $CLONE_LOG + git clone --recursive $GIT_OPTIONS $URL >> $CLONE_LOG 2>> $CLONE_LOG + r=$? + + if grep -q -E "(ssh_exchange_identification: read: Connection reset by peer|ssh_exchange_identification: Connection closed by remote host)" $CLONE_LOG + then + warning "Too many concurrent connections to the server. Retrying after sleep." + sleep $[$RANDOM % 5] + try_remote $URL + return $? + else + [ $r -eq 0 ] && rm -f $CLONE_LOG + return $r + fi +} + +function clone_repo +{ + NAME=$1 + CLONE_LOG=/tmp/clone-$BASHPID + if [ -e $NAME ] + then + info "Skipping $NAME -- already cloned. Use ./git_all.sh pull for update." + return 0 + fi + + info "Cloning repo $NAME." + for r in $REMOTES + do + URL="$r/$NAME" + try_remote $URL && debug "Success - $NAME cloned!" && break + done + + if [ ! -e $NAME ] + then + set +e + error "Failed to clone $NAME. Error was:" + set -e + cat $CLONE_LOG + rm -f $CLONE_LOG + return 1 + fi + + return 0 +} + +function pip_install +{ + # Use --user flag if not using a venv + if [ -n "$VIRTUAL_ENV" ]; then USER_FLAG="--user"; else USER_FLAG=""; fi + + debug "pip-installing: $@." + if ! pip3 install $PIP_OPTIONS $@ + then + error "pip failure ($@)." + fi +} + if [ "$INSTALL_REQS" -eq 1 ] then info "Installing dependencies..." @@ -271,68 +333,6 @@ if [ "$implementation" == "cpython" ]; then REPOS="${REPOS} $REPOS_CPYTHON"; fi # Install build dependencies until build isolation can be enabled pip install -U pip "setuptools==64.0.1" wheel cffi unicorn==2.0.1.post1 cmake ninja -function try_remote -{ - URL=$1 - debug "Trying to clone from $URL" - rm -f $CLONE_LOG - git clone --recursive $GIT_OPTIONS $URL >> $CLONE_LOG 2>> $CLONE_LOG - r=$? - - if grep -q -E "(ssh_exchange_identification: read: Connection reset by peer|ssh_exchange_identification: Connection closed by remote host)" $CLONE_LOG - then - warning "Too many concurrent connections to the server. Retrying after sleep." - sleep $[$RANDOM % 5] - try_remote $URL - return $? - else - [ $r -eq 0 ] && rm -f $CLONE_LOG - return $r - fi -} - -function clone_repo -{ - NAME=$1 - CLONE_LOG=/tmp/clone-$BASHPID - if [ -e $NAME ] - then - info "Skipping $NAME -- already cloned. Use ./git_all.sh pull for update." - return 0 - fi - - info "Cloning repo $NAME." - for r in $REMOTES - do - URL="$r/$NAME" - try_remote $URL && debug "Success - $NAME cloned!" && break - done - - if [ ! -e $NAME ] - then - set +e - error "Failed to clone $NAME. Error was:" - set -e - cat $CLONE_LOG - rm -f $CLONE_LOG - return 1 - fi - - return 0 -} - -function pip_install -{ - # Use --user flag if not using a venv - if [ -n "$VIRTUAL_ENV" ]; then USER_FLAG="--user"; else USER_FLAG=""; fi - - debug "pip-installing: $@." - if ! pip3 install $PIP_OPTIONS $@ - then - error "pip failure ($@)." - fi -} - info "Cloning angr components!" if [ $CONCURRENT_CLONE -eq 0 ] then From 66b146fab4b9a5f5479fe1bdbf1ea8addd12290c Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Wed, 7 Sep 2022 17:19:38 -0700 Subject: [PATCH 17/29] Replace pip3 with pip --- setup.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.sh b/setup.sh index d57f4146a..00f60c4b9 100755 --- a/setup.sh +++ b/setup.sh @@ -214,7 +214,7 @@ function pip_install if [ -n "$VIRTUAL_ENV" ]; then USER_FLAG="--user"; else USER_FLAG=""; fi debug "pip-installing: $@." - if ! pip3 install $PIP_OPTIONS $@ + if ! pip install $PIP_OPTIONS $@ then error "pip failure ($@)." fi @@ -279,7 +279,7 @@ then done if ! command -v workon &> /dev/null; then info "Could not find virtualenvwrapper preinstalled, installing via pip3..." - pip3 install --user virtualenvwrapper + pip install --user virtualenvwrapper set +e source ~/.local/bin/virtualenvwrapper.sh set -e @@ -323,7 +323,7 @@ then # older versions of pip will fail to process the --find-links arg silently # setuptools<64.0.1 is needed for editable installs for now, see angr/angr#3487 - pip3 install -U 'pip>=20.0.2' + pip install -U 'pip>=20.0.2' fi # Must happen after virutalenv is enabled to correctly detect python implementation @@ -413,7 +413,7 @@ then info "Installing some other helpful stuff" # we need the pyelftools from upstream - pip3 install -U ipython pylint ipdb nose nose-timer coverage flaky keystone-engine 'git+https://github.com/eliben/pyelftools#egg=pyelftools' + pip install -U ipython pylint ipdb nose nose-timer coverage flaky keystone-engine 'git+https://github.com/eliben/pyelftools#egg=pyelftools' echo '' info "All done! Execute \"workon $ANGR_VENV\" to use your new angr virtual" From dc06c939327dd45deb6f7c893147ae00d4309598 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Thu, 8 Sep 2022 16:47:51 -0700 Subject: [PATCH 18/29] Update ubuntu versions --- .github/workflows/setup-sh.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/setup-sh.yml b/.github/workflows/setup-sh.yml index 2f150b003..866133e1f 100644 --- a/.github/workflows/setup-sh.yml +++ b/.github/workflows/setup-sh.yml @@ -64,8 +64,8 @@ jobs: strategy: matrix: container: - - ubuntu:bionic - ubuntu:focal + - ubuntu:jammy - archlinux:latest - fedora:36 - almalinux:8 From 07b591e98f0a0a956a50569d4c5ffe9fcc63f7d2 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Mon, 12 Sep 2022 12:17:11 -0700 Subject: [PATCH 19/29] Add pip package to dependencies --- setup.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.sh b/setup.sh index 00f60c4b9..b45685f21 100755 --- a/setup.sh +++ b/setup.sh @@ -49,10 +49,10 @@ fi HOMEBREW_DEBS=${HOMEBREW_DEBS-git python3} # Linux distros -DEBS=${DEBS-git python3-dev python3-venv} -ARCHDEBS=${ARCHDEBS-git python base-devel} -RPMS=${RPMS-git python3-devel} -OPENSUSE_RPMS=${OPENSUSE_RPMS-git python3-devel} +DEBS=${DEBS-git python3-dev python3-venv python3-pip} +ARCHDEBS=${ARCHDEBS-git python python-pip base-devel} +RPMS=${RPMS-git python3-devel python3-pip} +OPENSUSE_RPMS=${OPENSUSE_RPMS-git python3-devel python3-pip} REPOS=${REPOS-archinfo pyvex cle claripy ailment angr angr-doc binaries} REPOS_CPYTHON=${REPOS_CPYTHON-angr-management} From 80dc52da9004b62dc7bc46a7e62e5ef2f2e425d1 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Mon, 12 Sep 2022 12:22:02 -0700 Subject: [PATCH 20/29] Use [[ in arch detection --- setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.sh b/setup.sh index b45685f21..af671fb4a 100755 --- a/setup.sh +++ b/setup.sh @@ -245,7 +245,7 @@ then $SUDO dnf install -yq $RPMS elif [[ "$ID $ID_LIKE" =~ "suse" ]]; then $SUDO zypper install -y $OPENSUSE_RPMS - elif [ "$ID $ID_LIKE" =~ "arch" ]; then + elif [[ "$ID $ID_LIKE" =~ "arch" ]]; then $SUDO pacman -Syq --noconfirm --needed $ARCHDEBS else error "We don't recognize this system. Please install equivelents of these debian packages: $DEBS" From 2ca770ff39f7e740322ce80f8881be5593112d95 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Mon, 12 Sep 2022 12:25:31 -0700 Subject: [PATCH 21/29] Remove almalinux 8 --- .github/workflows/setup-sh.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/setup-sh.yml b/.github/workflows/setup-sh.yml index 866133e1f..16adf373f 100644 --- a/.github/workflows/setup-sh.yml +++ b/.github/workflows/setup-sh.yml @@ -68,7 +68,6 @@ jobs: - ubuntu:jammy - archlinux:latest - fedora:36 - - almalinux:8 - almalinux:9 # OpenSUSE is disabled because they don't include tar # - opensuse/leap:15 From 1388667038cebd71a7e8e055447a60492b40849d Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Mon, 12 Sep 2022 12:33:11 -0700 Subject: [PATCH 22/29] Use virtualenvwrapper for linux --- .github/workflows/setup-sh.yml | 2 +- setup.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/setup-sh.yml b/.github/workflows/setup-sh.yml index 16adf373f..c05a73c8e 100644 --- a/.github/workflows/setup-sh.yml +++ b/.github/workflows/setup-sh.yml @@ -80,7 +80,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Run setup.sh - run: ./setup.sh -i -u + run: ./setup.sh -i -u -e angr - name: Test angr import run: python -c "import angr; print('angr imports!')" diff --git a/setup.sh b/setup.sh index af671fb4a..74323d87e 100755 --- a/setup.sh +++ b/setup.sh @@ -49,10 +49,10 @@ fi HOMEBREW_DEBS=${HOMEBREW_DEBS-git python3} # Linux distros -DEBS=${DEBS-git python3-dev python3-venv python3-pip} -ARCHDEBS=${ARCHDEBS-git python python-pip base-devel} -RPMS=${RPMS-git python3-devel python3-pip} -OPENSUSE_RPMS=${OPENSUSE_RPMS-git python3-devel python3-pip} +DEBS=${DEBS-git python3-dev python3-venv python3-pip virtualenvwrapper} +ARCHDEBS=${ARCHDEBS-git python python-pip base-devel python-virtualenvwrapper} +RPMS=${RPMS-git python3-devel python3-pip python3-virtualenvwrapper} +OPENSUSE_RPMS=${OPENSUSE_RPMS-git python3-devel python3-pip python3-virtualenvwrapper} REPOS=${REPOS-archinfo pyvex cle claripy ailment angr angr-doc binaries} REPOS_CPYTHON=${REPOS_CPYTHON-angr-management} From f1e92bb08822db9c7838cdd1fc8cd9b4471fda76 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Mon, 12 Sep 2022 12:39:00 -0700 Subject: [PATCH 23/29] Revert "Use virtualenvwrapper for linux" This reverts commit d1d58569033c6edb918844b1a618bd1142d53a0d. --- .github/workflows/setup-sh.yml | 2 +- setup.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/setup-sh.yml b/.github/workflows/setup-sh.yml index c05a73c8e..16adf373f 100644 --- a/.github/workflows/setup-sh.yml +++ b/.github/workflows/setup-sh.yml @@ -80,7 +80,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Run setup.sh - run: ./setup.sh -i -u -e angr + run: ./setup.sh -i -u - name: Test angr import run: python -c "import angr; print('angr imports!')" diff --git a/setup.sh b/setup.sh index 74323d87e..af671fb4a 100755 --- a/setup.sh +++ b/setup.sh @@ -49,10 +49,10 @@ fi HOMEBREW_DEBS=${HOMEBREW_DEBS-git python3} # Linux distros -DEBS=${DEBS-git python3-dev python3-venv python3-pip virtualenvwrapper} -ARCHDEBS=${ARCHDEBS-git python python-pip base-devel python-virtualenvwrapper} -RPMS=${RPMS-git python3-devel python3-pip python3-virtualenvwrapper} -OPENSUSE_RPMS=${OPENSUSE_RPMS-git python3-devel python3-pip python3-virtualenvwrapper} +DEBS=${DEBS-git python3-dev python3-venv python3-pip} +ARCHDEBS=${ARCHDEBS-git python python-pip base-devel} +RPMS=${RPMS-git python3-devel python3-pip} +OPENSUSE_RPMS=${OPENSUSE_RPMS-git python3-devel python3-pip} REPOS=${REPOS-archinfo pyvex cle claripy ailment angr angr-doc binaries} REPOS_CPYTHON=${REPOS_CPYTHON-angr-management} From 0f1fdf28031ba3a21ed9154eea298d8b3b328bd7 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Mon, 12 Sep 2022 12:54:01 -0700 Subject: [PATCH 24/29] Use venv for linux --- .github/workflows/setup-sh.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/setup-sh.yml b/.github/workflows/setup-sh.yml index 16adf373f..80709c71e 100644 --- a/.github/workflows/setup-sh.yml +++ b/.github/workflows/setup-sh.yml @@ -79,8 +79,17 @@ jobs: image: ${{ matrix.container }} steps: - uses: actions/checkout@v3 + - name: Install python3 + run: | + # Don't try to cover everything, just what's in CI + source /etc/os-release + [ "$ID" -eq "ubuntu" ] && apt update; apt install -y python3-venv + [ "$ID" -eq "arch" ] && pacman -Syu python + [ "$ID" -eq "fedora" ] && dnf install python3 + [ "$ID" -eq "almalinux" ] && yum install python3 + python3 -m venv venv - name: Run setup.sh - run: ./setup.sh -i -u + run: source venv/bin/activate && ./setup.sh -i -u - name: Test angr import run: python -c "import angr; print('angr imports!')" From f3fc6ebf9013c2fa5ffc79d5a50e444f7c9942dd Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Mon, 12 Sep 2022 12:55:17 -0700 Subject: [PATCH 25/29] Specify to run with bash --- .github/workflows/setup-sh.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/setup-sh.yml b/.github/workflows/setup-sh.yml index 80709c71e..1defdd35f 100644 --- a/.github/workflows/setup-sh.yml +++ b/.github/workflows/setup-sh.yml @@ -80,6 +80,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Install python3 + shell: bash run: | # Don't try to cover everything, just what's in CI source /etc/os-release From 353d21aafb07978027693d8003c75244bb694613 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Mon, 12 Sep 2022 12:56:55 -0700 Subject: [PATCH 26/29] Fix comparison --- .github/workflows/setup-sh.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/setup-sh.yml b/.github/workflows/setup-sh.yml index 1defdd35f..c6a96e06b 100644 --- a/.github/workflows/setup-sh.yml +++ b/.github/workflows/setup-sh.yml @@ -84,10 +84,10 @@ jobs: run: | # Don't try to cover everything, just what's in CI source /etc/os-release - [ "$ID" -eq "ubuntu" ] && apt update; apt install -y python3-venv - [ "$ID" -eq "arch" ] && pacman -Syu python - [ "$ID" -eq "fedora" ] && dnf install python3 - [ "$ID" -eq "almalinux" ] && yum install python3 + [ "$ID" = "ubuntu" ] && apt-get update; apt-get install -y python3-venv + [ "$ID" = "arch" ] && pacman -Syu python + [ "$ID" = "fedora" ] && dnf install python3 + [ "$ID" = "almalinux" ] && yum install python3 python3 -m venv venv - name: Run setup.sh run: source venv/bin/activate && ./setup.sh -i -u From 81a23e888b5139acd18cf976167a940b179d80f4 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Mon, 12 Sep 2022 12:59:18 -0700 Subject: [PATCH 27/29] Use . instead of source --- .github/workflows/setup-sh.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/setup-sh.yml b/.github/workflows/setup-sh.yml index c6a96e06b..71f8613e6 100644 --- a/.github/workflows/setup-sh.yml +++ b/.github/workflows/setup-sh.yml @@ -83,14 +83,14 @@ jobs: shell: bash run: | # Don't try to cover everything, just what's in CI - source /etc/os-release + . /etc/os-release [ "$ID" = "ubuntu" ] && apt-get update; apt-get install -y python3-venv [ "$ID" = "arch" ] && pacman -Syu python [ "$ID" = "fedora" ] && dnf install python3 [ "$ID" = "almalinux" ] && yum install python3 python3 -m venv venv - name: Run setup.sh - run: source venv/bin/activate && ./setup.sh -i -u + run: . venv/bin/activate && ./setup.sh -i -u - name: Test angr import run: python -c "import angr; print('angr imports!')" From 473247ef13cb39e1d24ac5e022fd1bfb9b52b701 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Tue, 10 Jan 2023 15:59:46 -0700 Subject: [PATCH 28/29] Fix remote detection not working if git is not already installed --- setup.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.sh b/setup.sh index af671fb4a..867bb0946 100755 --- a/setup.sh +++ b/setup.sh @@ -63,7 +63,11 @@ declare -A EXTRA_DEPS EXTRA_DEPS["angr"]="sqlalchemy unicorn==2.0.1.post1" EXTRA_DEPS["pyvex"]="--pre capstone" -ORIGIN_REMOTE=${ORIGIN_REMOTE-$(git remote -v | grep origin | head -n1 | awk '{print $2}' | sed -e "s|[^/:]*/angr-dev.*||")} +if [ ! $(which git) ]; then + ORIGIN_REMOTE=${ORIGIN_REMOTE-$(git remote -v | grep origin | head -n1 | awk '{print $2}' | sed -e "s|[^/:]*/angr-dev.*||")} +else + ORIGIN_REMOTE="https://github.com/" +fi REMOTES=${REMOTES-${ORIGIN_REMOTE}angr ${ORIGIN_REMOTE}shellphish ${ORIGIN_REMOTE}mechaphish https://git:@github.com/zardus https://git:@github.com/rhelmot https://git:@github.com/salls https://git:@github.com/lukas-dresel https://git:@github.com/mborgerson} From 399340108cd6557a43455e85594f337812770198 Mon Sep 17 00:00:00 2001 From: Kevin Phoenix Date: Tue, 10 Jan 2023 17:08:49 -0700 Subject: [PATCH 29/29] Fix conditional --- setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.sh b/setup.sh index 867bb0946..b8d7e310c 100755 --- a/setup.sh +++ b/setup.sh @@ -63,7 +63,7 @@ declare -A EXTRA_DEPS EXTRA_DEPS["angr"]="sqlalchemy unicorn==2.0.1.post1" EXTRA_DEPS["pyvex"]="--pre capstone" -if [ ! $(which git) ]; then +if [ ! -z '$(which git)' ]; then ORIGIN_REMOTE=${ORIGIN_REMOTE-$(git remote -v | grep origin | head -n1 | awk '{print $2}' | sed -e "s|[^/:]*/angr-dev.*||")} else ORIGIN_REMOTE="https://github.com/"