Merge branch 'main' into rust-polars-0.34

.Rbuildignore

@@ -21,3 +21,4 @@
 ^\.devcontainer$
 ^src/Makevars$
 ^tools/libr_polars\.a$
+^dev$

.github/workflows/check.yaml

@@ -132,3 +132,64 @@ jobs:
          print(getwd());
          source("./inst/misc/filter_rcmdcheck.R");
         shell: Rscript {0}
+
+  source-with-bin-check:
+    runs-on: ${{ matrix.os }}
+
+    name: ${{ matrix.os }} with pre-built binary (${{ matrix.r }})
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - macos-latest
+          - windows-latest
+          - ubuntu-latest
+        r:
+          - oldrel-1
+          - release
+          - devel
+        exclude:
+          - os: macos-latest
+            r: devel
+          - os: macos-latest
+            r: oldrel-1
+
+    env:
+      NOT_CRAN: "true"
+      LIB_SUMS_PATH: "tools/lib-sums.tsv"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check for pre-built binary
+        run: |
+          if [[ -f "${LIB_SUMS_PATH}" ]]; then
+            echo "TEST_BIN_LIB=true" >>"${GITHUB_ENV}"
+            rm -f "$(rustup which cargo)"
+          else
+            echo "TEST_BIN_LIB=false" >>"${GITHUB_ENV}"
+          fi
+
+      - uses: r-lib/actions/setup-pandoc@v2
+        if: env.TEST_BIN_LIB == 'true'
+
+      - uses: r-lib/actions/setup-r@v2
+        if: env.TEST_BIN_LIB == 'true'
+        with:
+          r-version: ${{ matrix.r }}
+          use-public-rspm: true
+          Ncpus: "2"
+
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        if: env.TEST_BIN_LIB == 'true'
+        with:
+          extra-packages: any::testthat, any::remotes
+          needs: check
+
+      - name: Install with pre-built binary
+        if: env.TEST_BIN_LIB == 'true'
+        shell: Rscript {0}
+        run: |
+          remotes::install_local(force = TRUE)
+          testthat::test_dir("tests")

.github/workflows/docs.yaml

@@ -10,10 +10,18 @@ on:
     branches:
       - main
     tags:
-      - "**" # Push events to every tag including hierarchical tags like v1.0/beta
+      - "v*"
   pull_request:
     branches:
       - main
+    paths:
+      - .github/workflows/docs.yaml
+      - docs/**
+      - R/**
+      - src/**
+      - vignetts/**
+      - DESCRIPTION
+      - README.md
   workflow_dispatch:
 
 concurrency:
@@ -22,6 +30,7 @@ concurrency:
 
 env:
   RPOLARS_FULL_FEATURES: "true"
+  LIBR_POLARS_BUILD: "true"
   RPOLARS_PROFILE: release
 
 jobs:

.github/workflows/release-lib.yaml

@@ -74,6 +74,7 @@ jobs:
           NOT_CRAN: "true"
           TARGET: ${{ matrix.target }}
           RPOLARS_PROFILE: ${{ env.RPOLARS_PROFILE }}
+          LIBR_POLARS_BUILD: "true"
         working-directory: src
         run: |
           LIB_PATH="$(pwd)/rust/target/${TARGET}/${RPOLARS_PROFILE}/${LIB_NAME}.a"
@@ -143,7 +144,7 @@ jobs:
           ARTIFACT_NAME="${LIB_NAME}-${LIB_VERSION}-${LIB_TARGET}.tar.gz"
           mkdir -p tools
           tar -xzf "libs/${ARTIFACT_NAME}" -C "tools"
-          rm -rf "libs"
+          rm -rf libs tools/lib-sums.tsv
           echo "LIBR_POLARS_PATH=$(pwd)/tools/${LIB_NAME}.a" >>"$GITHUB_ENV"
 
       - uses: r-lib/actions/setup-pandoc@v2

.github/workflows/release.yaml

@@ -3,7 +3,7 @@
 on:
   push:
     tags:
-      - "**" # Push events to every tag including hierarchical tags like v1.0/beta
+      - "v**" # Push events to every tag including hierarchical tags like v1.0/beta
   pull_request:
     branches:
       - main
@@ -25,6 +25,7 @@ defaults:
 env:
   RPOLARS_FULL_FEATURES: "true"
   RPOLARS_PROFILE: release-optimized
+  NOT_CRAN: "true"
 
 jobs:
   build:
@@ -42,40 +43,19 @@ jobs:
           - {os: macos-latest,   r: 'release'}
           - {os: windows-latest, r: 'release'}
           - {os: ubuntu-latest,   r: 'release'}
-          - {os: macos-latest,  r: 'release', target: 'x86_64-apple-darwin'}
-          - {os: macos-latest,  r: 'release', target: 'aarch64-apple-darwin'}
-          - {os: ubuntu-latest,  r: 'release', target: 'x86_64-unknown-linux-gnu'}
-          - {os: ubuntu-latest,  r: 'release', target: 'aarch64-unknown-linux-gnu'}
 
     steps:
       - uses: actions/checkout@v3
 
       - uses: ./.github/actions/setup
         with:
           rust-nightly: true
-          target: ${{ matrix.config.target }}
-
-      - name: Set for arm64 Linux
-        if: matrix.config.target == 'aarch64-unknown-linux-gnu'
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y gcc-aarch64-linux-gnu
-          echo 'CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc' >>"$GITHUB_ENV"
-
-      - name: Set build command
-        if: matrix.config.target != ''
-        run: echo "RPOLARS_BUILD_COMMAND_BASE=cargo build --target=${{ matrix.config.target }}" >>"$GITHUB_ENV"
-
-      - name: Fix path for Windows caching
-        if: runner.os == 'Windows'
-        run: echo "C:/Program Files/Git/usr/bin" >> $GITHUB_PATH
 
       - uses: r-lib/actions/setup-pandoc@v2
 
       - uses: r-lib/actions/setup-r@v2
         with:
           r-version: ${{ matrix.config.r }}
-          http-user-agent: ${{ matrix.config.http-user-agent }}
           use-public-rspm: true
           Ncpus: 2
 
@@ -111,37 +91,6 @@ jobs:
           }
         shell: Rscript {0}
 
-      - name: cross compile rust object files for another target arch
-        if: matrix.config.target != ''
-        run: |
-          export TARGET="${{ matrix.config.target }}"
-          LIB_PATH="$(pwd)/src/rust/target/${{ matrix.config.target }}/${{ env.RPOLARS_PROFILE }}/libr_polars.a"
-          pushd src
-          make -f Makevars.in "${LIB_PATH}"
-          popd
-          mkdir -p ./tools
-          mv "${LIB_PATH}" ./tools/
-          ls -l
-          ls -l ./tools/
-
-      - name: make source R package with pre-cross-compiled rust binaries for another target arch
-        if: matrix.config.target != ''
-        run: |
-          fine_name="$(R -s -e 'devtools::build(vignettes = FALSE, quiet = TRUE) |> cat()')"
-          new_file_name="$(echo $fine_name | sed "s/_.*/_cross_${{ matrix.config.target }}.tar.gz/")"
-          mv $fine_name $new_file_name
-          echo "new_file_name=${new_file_name}" >>"$GITHUB_ENV"
-
-      - name: unittest cross compilation, when host == target == 'x86_64-apple-darwin'
-        if: matrix.config.target == 'x86_64-apple-darwin' || matrix.config.target == 'x86_64-unknown-linux-gnu'
-        run: |
-          list.files()
-          polars_source_package_name = Sys.getenv("new_file_name")
-          install.packages(polars_source_package_name, repos = NULL, source = TRUE)
-          attach(getNamespace("polars")) # testthat assumes private namespace is in scope
-          testthat::test_dir("tests/testthat/")
-        shell: Rscript {0}
-
       - name: prep upload
         run: |
           mv ../polars* ./

DESCRIPTION

@@ -48,7 +48,9 @@ Config/Needs/website:
 Config/Needs/dev:
     brio,
     devtools,
+    dplyr,
     RcppTOML,
+    readr,
     rextendr,
     robinlovelace/styler.equals,
     spelling,
@@ -105,3 +107,4 @@ Collate:
     'zzz.R'
 Config/rextendr/version: 0.3.1
 VignetteBuilder: knitr
+Config/polars/LibVersion: 0.33.0

Makefile

@@ -57,7 +57,7 @@ install: ## Install the R package
 	&& R CMD INSTALL --no-multiarch --with-keep.source .
 
 .PHONY: all
-all: fmt build test README.md LICENSE.note ## build -> test -> Update README.md, LICENSE.note
+all: fmt tools/lib-sums.tsv build test README.md LICENSE.note ## build -> test -> Update README.md, LICENSE.note
 
 .PHONY: docs
 docs: build install README.md docs/docs/reference_home.md ## Generate docs
@@ -78,6 +78,10 @@ docs/docs/reference_home.md: docs/docs/reference_home.Rmd build ## Update the re
 LICENSE.note: src/rust/Cargo.lock ## Update LICENSE.note
 	Rscript -e 'rextendr::write_license_note(force = TRUE)'
 
+.PHONY: tools/lib-sums.tsv
+tools/lib-sums.tsv: ## Update the lib-sums.tsv file for pointing to the latest versions of the binary libraries
+	Rscript dev/generate-lib-sums.R
+
 .PHONY: test
 test: build install ## Run fast unittests
 	Rscript -e 'devtools::test()'

configure

@@ -39,6 +39,13 @@ check_bin_lib() {
     LIBR_POLARS_BUILD="false"
   fi
 
+  # On R-universe, we try to download the pre-built binary from GitHub releases.
+  if [ -z "${NOT_CRAN}" ] && [ -n "${MY_UNIVERSE}" ]; then
+    echo "It seems that this is on R-universe <${MY_UNIVERSE}>."
+    echo "Trying to download pre-built binary."
+    LIBR_POLARS_BUILD="false"
+  fi
+
   if [ "${LIBR_POLARS_BUILD}" = "false" ] && [ -f "tools/lib-sums.tsv" ] && [ ! -f "${LIBR_POLARS_PATH}" ] ; then
     echo ""
     echo "--------------- [ TRY TO DOWNLOAD PRE-BUILT BINARY ] ---------------"
@@ -64,6 +71,16 @@ check_bin_lib() {
       echo ""
     fi
     exit 0
+  elif [ "${LIBR_POLARS_BUILD}" = "false" ] && [ -f "${LIBR_POLARS_DEFAULT_PATH}" ]; then
+    echo ""
+    echo "---------------------- [LIBRARY BINARY FOUND] ----------------------"
+    echo "The library was not found at <${LIBR_POLARS_PATH}>,"
+    echo "but was found at <${LIBR_POLARS_DEFAULT_PATH}>."
+    echo "No need to build it."
+    echo "--------------------------------------------------------------------"
+    echo ""
+    sed -e "s|@RUST_TARGET@||" src/Makevars.in >src/Makevars
+    exit 0
   elif [ "${LIBR_POLARS_BUILD}" = "false" ]; then
     echo ""
     echo "-------------------- [LIBRARY BINARY NOT FOUND] --------------------"

dev/generate-lib-sums.R

@@ -0,0 +1,42 @@
+base_url = "https://github.com/pola-rs/r-polars/releases/download/"
+
+tag_prefix = "lib-v"
+
+lib_data_file_path = file.path("tools", "lib-sums.tsv")
+
+package_name = desc::desc_get("Package")
+current_lib_version = RcppTOML::parseTOML("src/rust/Cargo.toml")$package$version
+
+latest_released_lib_version = gert::git_remote_ls(remote = "https://github.com/pola-rs/r-polars/") |>
+  dplyr::pull(ref) |>
+  stringr::str_subset(stringr::str_c(r"(^refs/tags/)", tag_prefix)) |>
+  stringr::str_remove(stringr::str_c(".*", tag_prefix)) |>
+  numeric_version() |>
+  sort() |>
+  tail(1) |>
+  as.character()
+
+write_bin_lib_data = function(path, sums_url, libs_base_url) {
+  df = readr::read_table(sums_url, col_names = FALSE, show_col_types = FALSE) |>
+    dplyr::mutate(
+      url = glue::glue("{libs_base_url}{X2}"),
+      sha256sum = X1,
+      .keep = "none"
+    )
+
+  readr::write_tsv(df, path)
+}
+
+desc::desc_set(paste0("Config/", package_name, "/LibVersion"), current_lib_version)
+
+if (identical(current_lib_version, latest_released_lib_version)) {
+  message("Current lib version is available via the binary release.")
+  write_bin_lib_data(
+    lib_data_file_path,
+    glue::glue("{base_url}{tag_prefix }{latest_released_lib_version}/sha256sums.txt"),
+    glue::glue("{base_url}{tag_prefix }{latest_released_lib_version}/")
+  )
+} else {
+  message("Current lib version is not available via binary releases.")
+  if (fs::file_exists(lib_data_file_path)) fs::file_delete(lib_data_file_path)
+}

tools/lib-sums.tsv

@@ -0,0 +1,6 @@
+url	sha256sum
+https://github.com/pola-rs/r-polars/releases/download/lib-v0.33.0/libr_polars-0.33.0-aarch64-apple-darwin.tar.gz	697cca6d5f879f6cd9e7af147cfdc0df7b39ad38983bf7c75e2c3d060461b7b2
+https://github.com/pola-rs/r-polars/releases/download/lib-v0.33.0/libr_polars-0.33.0-aarch64-unknown-linux-gnu.tar.gz	c006edad4b68616cadbf3e4919f2c8693aa8fed4d6648c6759b9986d2a776fd6
+https://github.com/pola-rs/r-polars/releases/download/lib-v0.33.0/libr_polars-0.33.0-x86_64-apple-darwin.tar.gz	a6e40d6908b0f8797731b26fd7a8ba0c6f1ed0a3a7c6c0c8ce3b783af703f74a
+https://github.com/pola-rs/r-polars/releases/download/lib-v0.33.0/libr_polars-0.33.0-x86_64-pc-windows-gnu.tar.gz	06e4839296bdbbb4a32702671938e624e332f8555b067ee31f15b70d12cb762d
+https://github.com/pola-rs/r-polars/releases/download/lib-v0.33.0/libr_polars-0.33.0-x86_64-unknown-linux-gnu.tar.gz	d1b1545f88aaa9b558ecf447543f6ee3ca9bc0517c5fa8a84c7be5cbe99c12e0