Merge tiny GHA and refactors

test rhub

USING R-HUB IMAGES

simplify r test

modify rhub

fix docker dependencies

separate deps and check

run line

use of with

install dependencies

install pak

fix cd arguments

specify r user pkg dir

install openssl

dont use apt when it is not recognized

install rcmdcheck

explicit use of apt-get

correct cmd syntax

try script

remove multiple lines in gha

install rcmdcheck on docker

correct docker image call

simplify docker

docker: use local dir

use devtools with docker

try pak again

use devtools with docker

check with rcmdcheck on docker

install decor on docker

test with all rhub images

update pkg desc + add more devel R images

correct  clang versions

dont' install curl on docker

use old cache for ols OSes

downgrade checkout with ols OS

try checkout v2

try old OS again

remove old OSes for now

install pandoc on redhat

simplify gha

simplify gha more

skip R setup on docker

install pak

install pak via rscript

indent pak

fix asan errors?

full refactor + use libpng for mac m1

rpkg: use cpp/xxx.hpp instead of cpp11.hpp

consistent use of ThrowIfBad + add link to microdata

add microdata to main readme and pypkg

micro data in pypkg readme

.github/workflows/build-rpkg.yml

@@ -13,53 +13,92 @@ permissions: read-all
 jobs:
   R-CMD-check:
     runs-on: ${{ matrix.config.os }}
+    container: ${{ matrix.config.container }}
 
-    name: ${{ matrix.config.os }} (${{ matrix.config.r }})
+    name: ${{ matrix.config.name }}
 
     strategy:
       fail-fast: false
       matrix:
         config:
-          - {os: macos-latest,   r: 'release'}
-          - {os: windows-latest, r: 'release'}
-          - {os: ubuntu-latest,   r: 'release'}
+          - {os: macos-13, r: 'release', container: null, name: 'macos-13-r-release'}
+          - {os: macos-14, r: 'release', container: null, name: 'macos-14-r-release'}
+          - {os: ubuntu-latest, r: 'release', container: null, name: 'ubuntu-latest-r-release'}
+          - {os: windows-latest, r: 'release', container: null, name: 'windows-latest-r-release'}
+          - {os: macos-13, r: 'devel', container: null, name: 'macos-13-r-devel'}
+          - {os: macos-14, r: 'devel', container: null, name: 'macos-14-r-devel'}
+          - {os: ubuntu-latest, r: 'devel', container: null, name: 'ubuntu-latest-r-devel'}
+          - {os: windows-latest, r: 'devel', container: null, name: 'windows-latest-r-devel'}
+          - {os: ubuntu-latest, container: { image: 'ghcr.io/r-hub/containers/ubuntu-gcc12:latest' }, name: 'r-devel-linux-x86_64-debian-gcc'}
+          - {os: ubuntu-latest, container: { image: 'ghcr.io/r-hub/containers/ubuntu-clang:latest' }, name: 'r-devel-linux-x86_64-debian-clang'}
+          - {os: ubuntu-latest, container: { image: 'ghcr.io/r-hub/containers/atlas:latest' }, name: 'ATLAS'}
+          - {os: ubuntu-latest, container: { image: 'ghcr.io/r-hub/containers/c23:latest' }, name: 'C23'}
+          - {os: ubuntu-latest, container: { image: 'ghcr.io/r-hub/containers/clang-asan:latest' }, name: 'clang-ASAN'}
+          - {os: ubuntu-latest, container: { image: 'ghcr.io/r-hub/containers/clang16:latest' }, name: 'clang16'}
+          - {os: ubuntu-latest, container: { image: 'ghcr.io/r-hub/containers/clang17:latest' }, name: 'clang17'}
+          - {os: ubuntu-latest, container: { image: 'ghcr.io/r-hub/containers/clang18:latest' }, name: 'clang18'}
+          - {os: ubuntu-latest, container: { image: 'ghcr.io/r-hub/containers/clang19:latest' }, name: 'clang19'}
+          - {os: ubuntu-latest, container: { image: 'ghcr.io/r-hub/containers/clang20:latest' }, name: 'clang20'}
+          - {os: ubuntu-latest, container: { image: 'ghcr.io/r-hub/containers/donttest:latest' }, name: 'donttest'}
+          - {os: ubuntu-latest, container: { image: 'ghcr.io/r-hub/containers/gcc13:latest' }, name: 'gcc13'}
+          - {os: ubuntu-latest, container: { image: 'ghcr.io/r-hub/containers/gcc13:latest' }, name: 'gcc14'}
+          - {os: ubuntu-latest, container: { image: 'ghcr.io/r-hub/containers/intel:latest' }, name: 'intel'}
+          - {os: ubuntu-latest, container: { image: 'ghcr.io/r-hub/containers/mkl:latest' }, name: 'mkl'}
+          - {os: ubuntu-latest, container: { image: 'ghcr.io/r-hub/containers/nold:latest' }, name: 'noLD'}
+          - {os: ubuntu-latest, container: { image: 'ghcr.io/r-hub/containers/noremap:latest' }, name: 'noRemap'}
+          - {os: ubuntu-latest, container: { image: 'ghcr.io/r-hub/containers/valgrind:latest' }, name: 'rchk'}
+          - {os: ubuntu-latest, container: { image: 'ghcr.io/r-hub/containers/valgrind:latest' }, name: 'valgrind'}
 
     env:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
       R_KEEP_PKG_SOURCE: yes
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout
+        uses: actions/checkout@v4
 
-      - uses: r-lib/actions/setup-pandoc@v2
+      - name: Setup pandoc
+        if: matrix.config.container == null
+        uses: r-lib/actions/setup-pandoc@v2
 
-      - uses: r-lib/actions/setup-r@v2
+      - name: Setup R
+        if: matrix.config.container == null
+        uses: r-lib/actions/setup-r@v2
         with:
           r-version: ${{ matrix.config.r }}
           http-user-agent: ${{ matrix.config.http-user-agent }}
           use-public-rspm: true
 
       - name: Restore R package cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.cache/R
           key: ${{ runner.os }}-r-${{ matrix.config.r }}-cache
 
-      - uses: r-lib/actions/setup-r-dependencies@v2
-        with:
-          working-directory: './rpkg'
-          extra-packages: any::rcmdcheck
-          needs: check
+      - name: Install libpng (macOS)
+        if: matrix.config.os == 'macos-14' && matrix.config.r == 'devel'
+        run: |
+          curl -LO https://mac.r-project.org/bin/darwin20/arm64/libpng-1.6.38-darwin.20-arm64.tar.xz
+          sudo tar -xvf libpng-1.6.38-darwin.20-arm64.tar.xz -C /
+          echo "PKG_CONFIG_PATH=/opt/R/arm64/lib/pkgconfig" >> $GITHUB_ENV
 
-      - uses: r-lib/actions/check-r-package@v2
-        with:
-          working-directory: './rpkg'
-          upload-snapshots: true
-          build_args: 'c("--no-manual","--compact-vignettes=gs+qpdf")'
+      - name: Install dependencies
+        run: |
+          Rscript -e 'if (!requireNamespace("pak", quietly = TRUE)) { install.packages("pak", repos = "https://r-lib.github.io/p/pak/dev/") }'
+          Rscript -e 'pak::pkg_install("decor", dependencies = TRUE)'
+          Rscript -e 'pak::pkg_install("rcmdcheck", dependencies = TRUE)'
+          Rscript -e 'pak::pkg_install("pkgbuild", dependencies = TRUE)'
+          Rscript -e 'pak::pkg_install(".", dependencies = TRUE)'
+        working-directory: ./rpkg
+
+      - name: Check
+        run: |
+          Rscript -e 'pkgbuild::check_build_tools(debug = TRUE)'
+          Rscript -e 'rcmdcheck::rcmdcheck(args = c("--no-manual", "--as-cran"), build_args = "--no-manual", error_on = "error")'
+        working-directory: ./rpkg
 
       - name: Save R package cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.cache/R
           key: ${{ runner.os }}-r-${{ matrix.config.r }}-cache
-

README.md

@@ -30,6 +30,18 @@ This software is a full C++ ground-up rewrite of the original [Redatam Converter
 
 **For the Python package that allows to directly read REDATAM databases in Python, see the [pypkg](pypkg) directory.**
 
+**If you only need the processed data, you can download the [microdata repository](https://github.com/pachadotdev/redatam-microdata/releases). It is available in RDS format for easy loading into R.**
+
+**Available datasets:**
+
+- **Argentina: 1991, 2001, 2010**
+- **Bolivia: 2001, 2012**
+- **Chile: 2017**
+- **Ecuador: 2010**
+- **El Salvador: 2007**
+- **Guatemala: 2018**
+- **Mexico: 2000**
+
 ## Installation
 
 ### From binaries

dev/05-clear-gha-workflows.r

@@ -16,4 +16,4 @@ delete_old_workflow_runs <- function(owner, repo, keep = 1) {
 }
 
 # Call the function to delete old workflow runs, keeping only the latest 1 run
-delete_old_workflow_runs("pachadotdev", "open-redatam", keep = 1)
+delete_old_workflow_runs("pachadotdev", "open-redatam", keep = 5)

pypkg/README.md

@@ -35,6 +35,20 @@ rm -rf build dist redatam.egg-info
 pip install --use-pep517 .
 ```
 
+## Processed data
+
+If you only need the processed data, you can download the [microdata repository](https://github.com/pachadotdev/redatam-microdata/releases). It is available in RDS format for easy loading into R.
+
+Available datasets:
+
+- **Argentina**: 1991, 2001, 2010  
+- **Bolivia**: 2001, 2012  
+- **Chile**: 2017  
+- **Ecuador**: 2010  
+- **El Salvador**: 2007  
+- **Guatemala**: 2018  
+- **Mexico**: 2000  
+
 ## Requirements
 
 Python 3.8 or higher.

rpkg/.Rbuildignore

@@ -15,3 +15,11 @@
 ^cran-comments\.md$
 ^inst/extdata/galapagos$
 ^_pkgdown\.yml$
+^microdata$
+^redatam$
+^src/Makevars$
+^src/.*\.o$
+^src/.*\.so$
+^src/.*\.dll$
+^src/.*\.dylib$
+^libs$

rpkg/DESCRIPTION

@@ -1,7 +1,7 @@
 Package: redatam
 Type: Package
 Title: Import 'REDATAM' Files 
-Version: 2.0.1
+Version: 2.0.3
 Authors@R: c(
     person(
         given = "Mauricio",
@@ -32,14 +32,13 @@ Suggests:
     rmarkdown,
     testthat (>= 3.0.0)
 Depends: R(>= 3.5.0)
-Description: Import 'REDATAM' files into R. 'REDATAM' is a software for the
-    manipulation of microdata from censuses and surveys. This package provides
-    functions to import 'REDATAM' files into R. It is based on De Grande (2016)
+Description: Import 'REDATAM' formats into R via the 'Open REDATAM' C++ library
+    <https://github.com/litalbarkai/open-redatam> based on De Grande (2016)
     <https://www.jstor.org/stable/24890658>.
 License: Apache License (>= 2)
 URL: https://github.com/litalbarkai/open-redatam
 BugReports: https://github.com/litalbarkai/open-redatam/issues
-RoxygenNote: 7.3.1
+RoxygenNote: 7.3.2
 Encoding: UTF-8
 NeedsCompilation: yes
 VignetteBuilder: knitr

rpkg/README.md

@@ -13,14 +13,22 @@ For the standalone C++ command line application and desktop app, see the [main d
 To install the R package, run the following command:
 
 ```r
-install.packages("redatam")
+remotes::install_github("pachadotdev/open-redatam", subdir = "rpkg")
 ```
 
-Or install the development version from GitHub:
+## Processed data
 
-```r
-remotes::install_github("pachadotdev/open-redatam/rpkg", subdir = "rpkg")
-```
+If you only need the processed data, you can download the [microdata repository](https://github.com/pachadotdev/redatam-microdata/releases). It is available in RDS format for easy loading into R.
+
+Available datasets:
+
+- **Argentina**: 1991, 2001, 2010  
+- **Bolivia**: 2001, 2012  
+- **Chile**: 2017  
+- **Ecuador**: 2010  
+- **El Salvador**: 2007  
+- **Guatemala**: 2018  
+- **Mexico**: 2000  
 
 ## Requirements
 

rpkg/cleanup

@@ -0,0 +1,2 @@
+#!/bin/sh
+rm -f src/Makevars configure.log

rpkg/configure

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+PKG_CONFIG_NAME="redatam"
+
+# Check for pkg-config
+pkg-config --version >/dev/null 2>&1
+if [ $? -eq 0 ]; then
+  PKGCONFIG_CFLAGS=`pkg-config --cflags --silence-errors`
+  PKGCONFIG_LIBS=`pkg-config --libs`
+fi
+
+# Set default flags
+PKG_CFLAGS="-Ivendor -Iredatamlib -Iredatamlib/entities -Iredatamlib/exporters -Iredatamlib/readers"
+PKG_LIBS="-Lvendor -Lredatamlib"
+
+# Use pkg-config if available
+if [ "$PKGCONFIG_CFLAGS" ] || [ "$PKGCONFIG_LIBS" ]; then
+  echo "Found pkg-config cflags and libs!"
+  PKG_CFLAGS=${PKGCONFIG_CFLAGS}
+  PKG_LIBS=${PKGCONFIG_LIBS}
+fi
+
+# Uncomment CXXFLAGS in Makevars.in to debug
+# CXXFLAGS="-O0 -g -stdlib=libc++"
+LDFLAGS="-stdlib=libc++"
+
+# Write to Makevars using sed to replace placeholders
+sed -e "s|@cflags@|$PKG_CFLAGS|" \
+    -e "s|@libs@|$PKG_LIBS|" \
+    -e "s|@cxxflags@|$CXXFLAGS|" \
+    -e "s|@ldflags@|$LDFLAGS|" \
+    src/Makevars.in > src/Makevars
+
+# Success
+exit 0

rpkg/cran-comments.md

@@ -2,105 +2,4 @@
 
 0 errors | 0 warnings | 1 note
 
-* The reported CRAN errors on clang-asan/ubsan seem to come from a bug
-  external to the package
-* The bug was reported here
-  https://github.com/microsoft/DirectXShaderCompiler/issues/5971
-* The workaround is to set export ASAN_OPTIONS=alloc_dealloc_mismatch=0
-* After testing with clang-asan locally (see my clang setup at the end), I get
-  this
-
-```
-ℹ Testing redatam
-dotcode.c:1963:2: runtime error: call to function foo through pointer to incorrect function type 'void (*)(void *)'
-/tmp/RtmpawfzRb/foo.c:1: note: foo defined here
-SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior dotcode.c:1963:2 
-✔ | F W  S  OK | Context
-⠏ |          0 | galapagos                                                                                                   Opening dictionary file...
-Exporting cg15...
-Exporting provin...
-Exporting canton...
-Exporting parroq...
-Exporting zona...
-Fixing encoding...
-Tidying names...
-Tidying variables...
-⠋ |          1 | galapagos                                                                                                   Opening dictionary file...
-Exporting cg15...
-Exporting provin...
-Exporting canton...
-Exporting parroq...
-Exporting zona...
-Fixing encoding...
-Tidying names...
-Tidying variables...
-✔ |         16 | galapagos                                                                                                   
-
-══ Results ══════════════════════════════════════════════════════════════════════════════════════════════════════════════════
-[ FAIL 0 | WARN 0 | SKIP 0 | PASS 16 ]
-```
-
-# Clang setup
-
-```
-# install clang-19
-
-wget https://apt.llvm.org/llvm.sh -O dev/llvm.sh
-chmod +x dev/llvm.sh
-sudo ./dev/llvm.sh 19
-sudo apt install clang-19 lldb-19 libc++-dev libc++abi-dev
-sudo apt install libstdc++-12-dev
-
-export ASAN_OPTIONS=detect_leaks=0
-export RJAVA_JVM_STACK_WORKAROUND=0
-export RGL_USE_NULL=true
-export R_DONT_USE_TK=true
-
-sudo ln -s /usr/bin/clang-19 /usr/bin/clang
-sudo ln -s /usr/bin/clang++-19 /usr/bin/clang++
-
-# install R-devel from source
-
-sudo apt update
-sudo apt install -y build-essential gfortran libreadline-dev libx11-dev libxt-dev \
-                    libpng-dev libjpeg-dev libcairo2-dev xvfb libbz2-dev \
-                    libzstd-dev liblzma-dev libcurl4-openssl-dev \
-                    texinfo texlive texlive-fonts-extra \
-                    texlive-fonts-recommended texlive-latex-extra \
-                    libpcre2-dev libharfbuzz-dev libfribidi-dev \
-                    libclang-dev llvm-dev
-
-wget https://cran.r-project.org/src/base-prerelease/R-devel.tar.gz -O dev/R-devel.tar.gz
-cd dev
-tar -xzvf R-devel.tar.gz
-cd R-devel
-
-# add this to config.site
-## clang-ASAN, clang-UBSAN:
-CC="clang -fsanitize=address,undefined -fno-sanitize=float-divide-by-zero -fno-sanitize=alignment -fno-omit-frame-pointer"
-CXX="clang++ -fsanitize=address,undefined -fno-sanitize=float-divide-by-zero -fno-sanitize=alignment -fno-omit-frame-pointer -frtti -stdlib=libc++"
-CFLAGS="-g -O3 -Wall -pedantic"
-FFLAGS="-g -O2 -mtune=native"
-CXXFLAGS="-g -O3 -Wall -pedantic -stdlib=libc++"
-MAIN_LD="clang++ -fsanitize=undefined,address -stdlib=libc++"
-export ASAN_OPTIONS='detect_leaks=0:detect_odr_violation=0'
-export UBSAN_OPTIONS='print_stacktrace=1'
-export RGL_USE_NULL="true"
-export R_DONT_USE_TK="true"
-
-./configure \
-    --prefix=/opt/R/${R_VERSION} \
-    --enable-memory-profiling \
-    --enable-R-shlib \
-    --with-blas \
-    --with-lapack
-
-make
-
-dev/R-devel/bin/R
-# install.packages(c("knitr", "cpp11", "data.table", "stringi", "janitor", "rmarkdown", "devtools"))
-
-dev/R-devel/bin/R
-# devtools::clean_dll()
-# devtools::test()
-```
+* This was tested with all available R-Hub images.