the changes without b64 stuff

.github/workflows/ci-image-dev.yml

@@ -0,0 +1,45 @@
+name: CI - Development
+
+on:
+  workflow_dispatch:
+    inputs:
+      commitOrTag:
+        description: 'Commit or tag'
+        required: false
+        default: ''
+  push:
+    branches-ignore:
+      - 'release/**'
+
+jobs:
+  push-to-ecr:
+    permissions:
+      id-token: write
+      contents: write
+    name: build
+    runs-on: docker
+    steps:
+    - name: Clean environment
+      # Prune the Docker resources created over 10 days before the current execution (change the value for a more/less aggressive cleanup).
+      shell: bash
+      run: |
+        docker system df
+        docker system prune -a -f --filter "until=168h"
+        docker system df
+    - name: 'Checkout scm ${{ inputs.commitOrTag }}'
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        ref: ${{ inputs.commitOrTag }}
+    - name: Docker build and push
+      uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1
+      with:
+        file: docker/Dockerfile
+        push: true
+        tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla
+        tag-length: 8
+        registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com
+        aws-region: ${{ secrets.AWS_REGION_ZILLIQA }}
+        role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }}
+        oidc-role: ${{ secrets.OIDC_ROLE }}
+        cache-key: ${{ github.event.repository.name }}

.github/workflows/ci-image-release.yml

@@ -0,0 +1,66 @@
+name: CI - Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      commitOrTag:
+        description: 'Commit or tag'
+        required: false
+        default: ''
+  push:
+    tags:
+      - v*
+
+jobs:
+  release-image:
+    permissions:
+      id-token: write
+      contents: write
+    runs-on: docker
+    steps:
+    - name: 'Checkout scm ${{ inputs.commitOrTag }}'
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        ref: ${{ inputs.commitOrTag }}
+    - name: Check if the commit or tag was passed manually
+      id: set-tag
+      run: |
+        if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+          echo "tag=$(git rev-parse HEAD | cut -c1-8)" >> $GITHUB_OUTPUT
+        else
+          echo "tag=${{ github.ref_name }}" >> $GITHUB_OUTPUT
+        fi
+      shell: bash
+    - name: Check if the tag head is the same as master head
+      id: check-latest
+      if: github.event_name != 'workflow_dispatch'
+      run: |
+        if test $(git rev-parse origin/master) = $(git rev-parse HEAD); then
+          echo "latest=true" >> $GITHUB_OUTPUT
+        else
+          echo "latest=false" >> $GITHUB_OUTPUT
+        fi
+      shell: bash
+    - name: Docker build and push
+      uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1
+      with:
+        file: docker/Dockerfile
+        push: true
+        tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla:${{ steps.set-tag.outputs.tag }}
+        tag-latest: ${{ steps.check-latest.outputs.latest }}
+        registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com
+        aws-region: ${{ secrets.AWS_REGION_ZILLIQA }}
+        role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }}
+        oidc-role: ${{ secrets.OIDC_ROLE }}
+        cache-key: ${{ github.event.repository.name }}
+    - name: Docker build and push (Dockerhub)
+      uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1
+      with:
+        file: docker/Dockerfile
+        push: true
+        tag: Zilliqa/scilla:${{ steps.set-tag.outputs.tag }}
+        tag-latest: ${{ steps.check-latest.outputs.latest }}
+        cache-key: ${{ github.event.repository.name }}
+        registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
+        registry-password: ${{ secrets.DOCKERHUB_PASSWORD }}

.github/workflows/ci-image-test.yml

@@ -0,0 +1,63 @@
+name: CI - Integration tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      commitOrTag:
+        description: 'Commit or tag'
+        required: false
+        default: ''
+  pull_request:
+    branches:
+      - 'master'
+
+jobs:
+  run-tests:
+    permissions:
+      id-token: write
+      contents: write
+    name: tests
+    runs-on: docker
+    steps:
+    - name: Clean environment
+      # Prune the Docker resources created over 10 days before the current execution (change the value for a more/less aggressive cleanup).
+      shell: bash
+      run: |
+        docker system df
+        docker system prune -a -f --filter "until=336h"
+        docker system df
+    - name: 'Checkout scm ${{ inputs.commitOrTag }}'
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        ref: ${{ inputs.commitOrTag }}
+    - name: Configure AWS Credentials
+      uses: Zilliqa/gh-actions-workflows/actions/configure-aws-credentials@v1
+      with:
+        role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }}
+        oidc-role: ${{ secrets.OIDC_ROLE }}
+        aws-region: ${{ secrets.AWS_REGION_ZILLIQA }}
+    - name: Login to the registry
+      uses: docker/login-action@v2
+      with:
+        registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com
+    - name: Build Docker images
+      run: |
+        DOCKER_BUILDKIT=1 docker build -t scilla:tests --build-arg ACCOUNT_ID=${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }} -f docker/Dockerfile.test .
+      shell: bash
+    - name: Run make test
+      run: |
+        docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test'
+      shell: bash
+    - name: Run make test_server
+      run: |
+        docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server'
+      shell: bash
+    - name: Run make coveralls
+      run: |
+        docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make coveralls TRAVIS_JOB_ID=${{ github.run_number }}'
+      shell: bash
+    - name: Run make lint
+      run: |
+        docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint'
+      shell: bash

Makefile

@@ -170,7 +170,7 @@ clean:
 	dune clean
 # Remove remaining files/folders ignored by git as defined in .gitignore (-X)
 # but keeping a local opam switch and other dependencies built.
-	git clean -dfXq --exclude=\!deps/** --exclude=\!_opam/** --exclude=\!_esy/** --exclude=\!vcpkg_installed
+	git clean -dfXq --exclude=\!deps/** --exclude=\!_opam/** --exclude=\!_esy/** --exclude=\!vcpkg_installed --exclude=\!vcpkg_installed/**
 
 # Clean up libff installation
 cleanall: clean

docker/Dockerfile

@@ -1,7 +1,10 @@
 # escape=\
 
 # Common dependencies of the builder and runner stages.
-FROM ubuntu:22.04 AS base
+FROM --platform=linux/amd64 ubuntu:22.04 AS base
+
+RUN apt-get update -y
+
 # Format guideline: one package per line and keep them alphabetically sorted
 RUN apt-get update -y \
     && apt-get install -y software-properties-common \
@@ -92,6 +95,9 @@ COPY scilla.opam .
 COPY shell.nix .
 COPY .ocamlformat .
 COPY tests/ tests
+COPY handle_connection.sh .
+COPY run_scilla_tcp.sh .
+COPY write_files.py .
 
 # Make sure vcpkg installs brings in the dependencies
 RUN --mount=type=cache,target=/root/.cache/vcpkg/ ${VCPKG_ROOT}/vcpkg install --triplet=x64-linux-dynamic
@@ -115,10 +121,14 @@ RUN mkdir -p ${VCPKG_INSTALL_LIB_DIR} \
   && rm -rf vcpkg_installed \
   && ln -s ${BUILD_DIR}/vcpkg_installed vcpkg_installed
 
-FROM ubuntu:22.04
+FROM --platform=linux/amd64 ubuntu:22.04
 
 RUN apt-get update -y \
     && apt-get install -y build-essential \
+    netcat-openbsd \
+    vim \
+    socat \
+    python3 \
     libgmp-dev
 
 ARG SOURCE_DIR="/scilla/${MAJOR_VERSION}"

docker/Dockerfile.test

@@ -0,0 +1,17 @@
+ARG ACCOUNT_ID
+
+FROM ${ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com/scilla:429e2f9
+
+ENV VCPKG_ROOT="/vcpkg"
+ENV SCILLA_REPO_ROOT="/scilla/0"
+
+WORKDIR /scilla/0/
+COPY . /scilla/0/
+
+RUN apt update \
+  && apt install -y sudo
+
+RUN eval $(opam env) \
+  && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes
+
+RUN ./scripts/install_shellcheck_ubuntu.sh

handle_connection.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+echo "Starting server." 1>&2
+
+# Start scilla-server in the background
+rm -rf /tmp/scilla-socket/server.sock
+/scilla/0/bin/scilla-server -socket /tmp/scilla-socket/server.sock 1>&2 &
+
+# Get the PID of scilla-server
+SERVER_PID=$!
+
+echo "forwarding traffic." 1>&2
+
+# Block until we see the scilla server has provided the socket
+while [ ! -S /tmp/scilla-socket/server.sock ]; do sleep 1; done
+
+echo "forwarding other traffic." 1>&2
+
+# Forward traffic from stdin (provided by socat) to Unix socket.
+# Note this means we ONLY use stderr when echoing within this script to avoid it going to the socket
+socat - UNIX-CONNECT:/tmp/scilla-socket/server.sock
+
+echo "connection closed - killing" 1>&2
+echo "" 1>&2
+
+# Kill the scilla-server process when done
+kill $SERVER_PID

run_scilla_tcp.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+echo "Starting server." 1>&2
+
+rm -rf myfifo 1>&2
+mkfifo myfifo 1>&2
+
+# Clear the directories we will be using the files for
+mkdir -p /tmp/scilla_init/
+mkdir -p /tmp/scilla_input/
+
+rm -rf /tmp/scilla_init/*
+rm -rf /tmp/scilla_input/*
+
+# netcat listen to 12346 and forward it to/from a local socket (this is the state queries/updates)
+nc -lkv 12346 <myfifo | nc -lkUD /tmp/scilla-server.sock > myfifo &
+
+
+# socat - every connection on 12345, run the script handle_connection and pipe the input to it (this is the initial run/create/etc commands)
+socat TCP-LISTEN:12347,reuseaddr,fork EXEC:"/scilla/0/write_files.py",nofork &
+socat TCP-LISTEN:12345,reuseaddr,fork EXEC:"/scilla/0/handle_connection.sh",nofork

src/base/cpp/config/discover.ml

@@ -30,7 +30,9 @@ let () =
           We rely on a simple python program (in vcpkg_cmd) to do so portably.
       *)
       let vcpkg_cmd =
-        "python3 -c 'import os, sys; print(os.path.realpath(os.environ[\"SCILLA_REPO_ROOT\"]" ^^ "+\"/%s\"))'"
+        "python3 -c 'import os, sys; \
+         print(os.path.realpath(os.environ[\"SCILLA_REPO_ROOT\"]"
+        ^^ "+\"/%s\"))'"
       in
       let vcpkg_triplet_path =
         input_line

write_files.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+
+def main():
+    # Read the input as binary data from stdin
+    input_data = sys.stdin.buffer.read()
+
+    # Split the input into a path and data
+    try:
+        path, data = input_data.split(b' ', 1)
+    except ValueError:
+        print("Invalid input format. Please provide a valid absolute path and data.")
+        sys.exit(1)
+
+    # Ensure the path is absolute
+    if not os.path.isabs(path.decode()):
+        print("Please provide an absolute path.")
+        sys.exit(1)
+
+    # Write binary data to the specified file
+    try:
+        with open(path.decode(), 'wb') as file:
+            file.write(data)
+        print(f"Data written to {path.decode()}")
+    except Exception as e:
+        print(f"Error writing data to {path.decode()}: {str(e)}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()