bump

.buildconfig

@@ -1,2 +1,2 @@
 PG_VERSION=16.4
-SDK_VERSION=3.1.69.10bi
+SDK_VERSION=3.1.70.4bi

cibuild.sh

@@ -2,7 +2,7 @@
 
 # data transfer zone this is == (wire query size + result size ) + 2
 # expressed in EMSDK MB
-export CMA_MB=${CMA_MB:-64}
+export CMA_MB=${CMA_MB:-32}
 
 export CI=${CI:-false}
 
@@ -19,10 +19,16 @@ export DEBUG=${DEBUG:-false}
 export PGDATA=${PGROOT}/base
 export PGUSER=${PGUSER:-postgres}
 export PGPATCH=${WORKSPACE}/patches
-export TOTAL_MEMORY=${TOTAL_MEMORY:-128MB}
+export TOTAL_MEMORY=${TOTAL_MEMORY:-512MB}
 export WASI=${WASI:-false}
+# 72 - 144228352
+# -sINITIAL_HEAP not compatible with IMPORTED_MEMORY (which is enabled indirectly via SHARED_MEMORY, RELOCATABLE, ASYNCIFY_LAZY_LOAD_CODE)
+export MEMORY="-sINITIAL_MEMORY=128MB -sMAXIMUM_MEMORY=${TOTAL_MEMORY} -sSTACK_SIZE=2MB -sGLOBAL_BASE=${CMA_MB}MB -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH"
+#export MEMORY="-sTOTAL_MEMORY=${TOTAL_MEMORY} -sSTACK_SIZE=2MB -sGLOBAL_BASE=${CMA_MB}MB -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH"
 
 
+export PYDK_CFLAGS="-Wno-missing-prototypes"
+
 # exit on error
 EOE=false
 
@@ -78,7 +84,7 @@ then
 else
     if which emcc
     then
-        echo "emcc found in PATH=$PATH"
+        echo "emcc found in PATH=$PATH (please set PREFIX)"
     else
         . /opt/python-wasm-sdk/wasm32-bi-emscripten-shell.sh
     fi
@@ -92,9 +98,9 @@ else
         node : $(which node) $($(which node) -v)
         PNPM : $(which pnpm)
 
+    PREFIX=$PREFIX
 
 "
-
     # custom code for node/web builds that modify pg main/tools behaviour
     # this used by both node/linkweb build stages
 
@@ -219,10 +225,17 @@ END
 #define PGDEBUG 0
 #endif
 END
+    cat >> ${PG_DEBUG_HEADER} <<END
+#if defined(PG_VERSION_STR)
+#undef PG_VERSION_STR
+#endif
+#define PG_VERSION_STR "PostgreSQL ${PG_VERSION} (PGlite PGLITE_VERSION) on wasm32"
+END
+
     fi
 
     mkdir -p ${PGROOT}/include/postgresql/server
-    cp ${PG_DEBUG_HEADER} ${PGROOT}/include/
+    #cp ${PG_DEBUG_HEADER} ${PGROOT}/include/
     cp ${PG_DEBUG_HEADER} ${PGROOT}/include/postgresql
     cp ${PG_DEBUG_HEADER} ${PGROOT}/include/postgresql/server
 
@@ -376,8 +389,20 @@ then
             fi
         fi
         echo "======================= ${extra_ext} : $(pwd) ==================="
+        if [ -f ./extra/${extra_ext}.sh ]
+        then
+            ./extra/${extra_ext}.sh || exit 400
+        else
+            echo "
+
+    WARNING: Current source tree has not support ./extra/${extra_ext}.sh
+             for building ${extra_ext}
+
+
+
+"
+        fi
 
-        ./extra/${extra_ext}.sh || exit 400
 
         python3 cibuild/pack_extension.py
     done

cibuild/getsyms.py

@@ -8,7 +8,6 @@ def dbg(*argv, **kw):
     return print(*argv,**kw)
 
 
-SNIFF="clock_gettime"
 SNIFF=""
 
 
@@ -89,7 +88,17 @@ def dbg(*argv, **kw):
 
         if SNIFF:
             if line.find(SNIFF)>=0:
-                dbg(line)
+                dbg(f"""
+
+-------------------------------------------------------------------------------------
+
+{line=}
+{typ=}
+
+
+-------------------------------------------------------------------------------------
+
+""")
 
         try:
             if typ in ('def','var'):
@@ -129,7 +138,7 @@ def dbg(*argv, **kw):
                 elif left.find('::')> 0:
                     if VERBOSE:
                         raise Exception("bad export (c++)")
-                    continue
+                    #continue
                 elif left.find(' ')> 0:
                     if VERBOSE:
                         raise Exception("bad export (space)")

cibuild/linkexport.sh

@@ -1,10 +1,12 @@
 # this only runs when wasm-objdump is working and OBJDUMP not set to false
 
 echo "============= link export : begin ==============="
+
+
 emcc $EMCC_WEB -fPIC -sMAIN_MODULE=1 -O0 \
  -D__PYDK__=1 -DPREFIX=${PGROOT} \
- -sTOTAL_MEMORY=${TOTAL_MEMORY} -sEXPORT_ALL -sSTACK_SIZE=4MB -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sGLOBAL_BASE=${CMA_MB}MB \
- -sERROR_ON_UNDEFINED_SYMBOLS -sASSERTIONS=0 \
+ $MEMORY \
+ -sEXPORT_ALL -sERROR_ON_UNDEFINED_SYMBOLS -sASSERTIONS=0 \
  -lnodefs.js -lidbfs.js \
  -sEXPORTED_RUNTIME_METHODS=FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack,ccall,cwrap,callMain \
  $PGPRELOAD \

cibuild/linkimports.sh

@@ -59,25 +59,30 @@ _getenv
 _interactive_one
 _interactive_read
 _interactive_write
+_loop
 _lowerstr
 _main
-_main_repl
 _pg_initdb
 _pg_repl_raf
 _pg_shutdown
 _readstoplist
 _searchstoplist
 _setenv
+_setup
 _shmem_request_hook
 _shmem_startup_hook
 _stderr
 _TopMemoryContext
+__ZNSt12length_errorD1Ev
+__ZNSt20bad_array_new_lengthD1Ev
+__ZNSt16invalid_argumentD1Ev
 """.splitlines():
     if sym and not sym in matches:
         matches.append(sym)
 
-#__ZNSt13runtime_errorD1Ev
-
+# __ZNSt13runtime_errorD1Ev  : i32 std::length_error::~length_error(i32)
+# __ZNSt20bad_array_new_lengthD1Ev : std::bad_array_new_length::~bad_array_new_length()
+# __ZNSt16invalid_argumentD1Ev : std::invalid_argument::~invalid_argument()
 matches.sort()
 
 for sym in matches:

cibuild/linkwasi.sh

@@ -7,6 +7,7 @@ WASI_CFLAGS="-DPATCH_PG_DEBUG=/tmp/pglite/include/pg_debug.h -DPREFIX=/tmp/pglit
  -Ipostgresql/src/backend \
  -c -o build/postgres/wasi_dlfcn.o \
  -Ibuild/postgres/src/include patches/wasi_dlfcn.c || exit 8
+
 #  -L./build/postgres/src/backend/snowball -ldict_snowball
 # ./build/postgres/src/backend/snowball/dict_snowball.o
 # ./build/postgres/src/backend/snowball/libdict_snowball.a
@@ -770,7 +771,7 @@ llvm-ar cr ../../libpglite.a $PGOBJ
 
 # just linking
 
-
+# -Wl,--no-entry -mexec-model=reactor
 
 $CC -o postgres \
  -fno-strict-aliasing \
@@ -784,11 +785,13 @@ $CC -o postgres \
     ../../src/pl/plpgsql/src/libplpgsql.a \
  -lz -lm -lwasi-emulated-mman -lwasi-emulated-signal -lc \
  -Wl,--export=pg_initdb \
+ -Wl,--export=setup \
+ -Wl,--export=loop \
  -Wl,--export=interactive_one \
  -Wl,--export=use_socketfile \
  -Wl,--export=interactive_write \
  -Wl,--export=interactive_read
- -Wl,--global-base=33554432
+ -Wl,--global-base=33333333
 
 cp -vf postgres postgres.wasi || exit 192
 #cp -vf postgres.wasi /tmp/pglite/bin/postgres.wasi

cibuild/pgbuild.sh

@@ -32,26 +32,30 @@ CC_PGLITE=$CC_PGLITE
 # TODO: --with-libxslt   add to sdk
 #  --disable-atomics https://github.com/WebAssembly/threads/pull/147  "Allow atomic operations on unshared memories"
 
-if ${WASI}
-then
-    echo "WASI BUILD: turning off xml/xslt support"
-    XML2=""
-    UUID=""
-    BUILD=wasi
-    export MAIN_MODULE="-lwasi-emulated-getpid -lwasi-emulated-mman -lwasi-emulated-signal -lwasi-emulated-process-clocks"
-else
-    if $CI
+
+
+    if ${WASI}
     then
-        # do not build obsolete ext xml2 on CI
-        XML2="--with-zlib --with-libxml"
+        echo "WASI BUILD: turning off xml/xslt support"
+        XML2=""
+        UUID=""
+        BUILD=wasi
+        export MAIN_MODULE="-lwasi-emulated-getpid -lwasi-emulated-mman -lwasi-emulated-signal -lwasi-emulated-process-clocks"
     else
-        XML2="--with-zlib --with-libxml --with-libxslt"
+        # --with-libxml does not fit with --without-zlib
+        if $CI
+        then
+            # do not build obsolete ext xml2 on CI
+            XML2="--with-zlib --with-libxml"
+        else
+            XML2="--with-zlib --with-libxml --with-libxslt"
+        fi
+        UUID="--with-uuid=ossp"
+        BUILD=emscripten
+        export MAIN_MODULE="-sMAIN_MODULE=1"
     fi
-    UUID="--with-uuid=ossp"
-    BUILD=emscripten
-    export MAIN_MODULE="-sMAIN_MODULE=1"
-fi
-# --with-libxml does not fit with --without-zlib
+
+
 
     export XML2_CONFIG=$PREFIX/bin/xml2-config
     export ZIC=$(pwd)/bin/zic
@@ -76,6 +80,14 @@ fi
     mkdir -p bin
 
 
+    [ -f /usr/bin/zic ] && cp /usr/bin/zic bin/
+    if [ -f bin/zic ]
+    then
+        echo "using system zic"
+        GETZIC=false
+    else
+        GETZIC=true
+    fi
 
 
     if $WASI
@@ -84,26 +96,31 @@ fi
         cat > ${PGROOT}/config.site <<END
 ac_cv_exeext=.wasi
 END
-        cat > bin/zic <<END
+        if $GETZIC
+        then
+            cat > bin/zic <<END
 #!/bin/bash
 #. /opt/python-wasm-sdk/wasm32-wasi-shell.sh
-TZ=UTC PGTZ=UTC wasi-run $(pwd)/src/timezone/zic.wasi \$@
+TZ=UTC PGTZ=UTC $(command -v wasi-run) $(pwd)/src/timezone/zic.wasi \$@
 END
+        fi
 
     else
         export EXT=wasm
         cat > ${PGROOT}/config.site <<END
 ac_cv_exeext=.cjs
 END
-        cat > bin/zic <<END
+
+        if $GETZIC
+        then
+            cat > bin/zic <<END
 #!/bin/bash
 #. /opt/python-wasm-sdk/wasm32-bi-emscripten-shell.sh
-TZ=UTC PGTZ=UTC node $(pwd)/src/timezone/zic.cjs \$@
+TZ=UTC PGTZ=UTC $(command -v node) $(pwd)/src/timezone/zic.cjs \$@
 END
+        fi
     fi
 
-
-
     if EM_PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig CONFIG_SITE=${PGROOT}/config.site emconfigure $CNF --with-template=$BUILD
     then
         echo configure ok
@@ -130,25 +147,26 @@ END
 
     > /tmp/disable-shared.log
 
-    cat > bin/emsdk-shared <<END
+    cat > $PGROOT/bin/emsdk-shared <<END
 #!/bin/bash
 echo "[\$(pwd)] $0 \$@" >> /tmp/disable-shared.log
 # shared build
 \${PG_LINK:-emcc} -L${PREFIX}/lib -DPREFIX=${PGROOT} -shared -sSIDE_MODULE=1 \$@ -Wno-unused-function
 END
+    ln -sf $PGROOT/bin/emsdk-shared bin/emsdk-shared
+
 
-    cat > bin/wasi-shared <<END
+    cat > $PGROOT/bin/wasi-shared <<END
 #!/bin/bash
 echo "[\$(pwd)] $0 \$@" >> /tmp/disable-shared.log
 # shared build
 echo ===================================================================================
 wasi-c -L${PREFIX}/lib -DPREFIX=${PGROOT} -shared \$@ -Wno-unused-function
 echo ===================================================================================
 END
+    ln -sf $PGROOT/bin/wasi-shared bin/wasi-shared
 
-
-
-    chmod +x bin/zic bin/wasi-shared bin/emsdk-shared
+    chmod +x bin/zic $PGROOT/bin/wasi-shared $PGROOT/bin/emsdk-shared
 
     # for zic and emsdk-shared/wasi-shared called from makefile
     export PATH=$(pwd)/bin:$PATH
@@ -168,7 +186,7 @@ END
 
     # only required for static initdb
     EMCC_CFLAGS="-sERROR_ON_UNDEFINED_SYMBOLS=1 ${CC_PGLITE}"
-    EMCC_CFLAGS="${EMCC_CFLAGS} -sTOTAL_MEMORY=${TOTAL_MEMORY} -sSTACK_SIZE=5MB -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sGLOBAL_BASE=${CMA_MB}MB"
+    EMCC_CFLAGS="${EMCC_CFLAGS} $MEMORY"
     EMCC_CFLAGS="${EMCC_CFLAGS} -DPREFIX=${PGROOT}"
 
     EMCC_CFLAGS="${EMCC_CFLAGS} -Wno-macro-redefined -Wno-unused-function"
@@ -243,6 +261,12 @@ USER="${PGPASS:-postgres}"
 PASS="${PGUSER:-postgres}"
 md5pass =  "md5" + __import__('hashlib').md5(USER.encode() + PASS.encode()).hexdigest()
 print(f"localhost:5432:postgres:{USER}:{md5pass}")
+
+USER="postgres"
+PASS="postgres"
+md5pass =  "md5" + __import__('hashlib').md5(USER.encode() + PASS.encode()).hexdigest()
+print(f"localhost:5432:postgres:{USER}:{md5pass}")
+
 USER="login"
 PASS="password"
 md5pass =  "md5" + __import__('hashlib').md5(USER.encode() + PASS.encode()).hexdigest()