Tdrz/be pgcrypto

.gitignore

@@ -56,3 +56,4 @@ pglite.js
 pglite.html
 *.map
 pglite-wasm/excluded.imports
+*.symbols

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "pglite/pgtap"]
 	path = pglite/pgtap
 	url = https://github.com/theory/pgtap.git
+[submodule "pglite/pg_uuidv7"]
+	path = pglite/pg_uuidv7
+	url = https://github.com/fboulnois/pg_uuidv7/

build-pglite.sh

@@ -8,13 +8,15 @@
 INSTALL_FOLDER=${INSTALL_FOLDER:-"/install/pglite"}
 
 # build with optimizations by default aka release
-PGLITE_CFLAGS="-O2"
+# PGLITE_CFLAGS="-sDYLINK_DEBUG=2 -g -gsource-map --no-wasm-opt -Wbad-function-cast -Wcast-function-type"
+PGLITE_CFLAGS=""
 if [ "$DEBUG" = true ]
 then
     echo "pglite: building debug version."
-    PGLITE_CFLAGS="-g -gsource-map --no-wasm-opt"
+    PGLITE_CFLAGS="$PGLITE_CFLAGS -g -gsource-map --no-wasm-opt --emit-symbol-map"
 else
     echo "pglite: building release version."
+    PGLITE_CFLAGS="$PGLITE_CFLAGS -O2"
     # we shouldn't need to do this, but there's a bug somewhere that prevents a successful build if this is set
     unset DEBUG
 fi
@@ -38,7 +40,7 @@ else
 fi
 
 # we define here "all" emscripten flags in order to allow native builds (like libpglite)
-EXPORTED_RUNTIME_METHODS="addFunction,removeFunction,FS,MEMFS"
+EXPORTED_RUNTIME_METHODS="addFunction,removeFunction,FS,MEMFS,wasmTable"
 PGLITE_EMSCRIPTEN_FLAGS="-sWASM_BIGINT \
 -sSUPPORT_LONGJMP=emscripten \
 -sFORCE_FILESYSTEM=1 \
@@ -65,11 +67,14 @@ emmake make PORTNAME=emscripten -j || { echo 'error: emmake make PORTNAME=emscri
 emmake make PORTNAME=emscripten install || { echo 'error: emmake make PORTNAME=emscripten install' ; exit 22; }
 
 # Step 3.1: make all contrib extensions - do not install
+
+# Step 3.1.2 all the rest of contrib
 emmake make PORTNAME=emscripten -C contrib/ -j || { echo 'error: emmake make PORTNAME=emscripten -C contrib/ -j' ; exit 31; }
 # Step 3.2: make dist contrib extensions - this will create an archive for each extension
 emmake make PORTNAME=emscripten -C contrib/ dist || { echo 'error: emmake make PORTNAME=emscripten -C contrib/ dist' ; exit 32; }
 # the above will also create a file with the imports that each extension needs - we pass these as input in the next step for emscripten to keep alive
 
+
 # Step 4: make and dist other extensions
 SAVE_PATH=$PATH
 PATH=$PATH:$INSTALL_FOLDER/bin
@@ -78,7 +83,7 @@ emmake make OPTFLAGS="" PORTNAME=emscripten -C pglite/ dist || { echo 'error: ma
 PATH=$SAVE_PATH
 
 # Step 5: make and install pglite
-EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack,addFunction,removeFunction"
+EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack,addFunction,removeFunction,wasmTable"
 PGLITE_EMSCRIPTEN_FLAGS="-sWASM_BIGINT \
 -sSUPPORT_LONGJMP=emscripten \
 -sFORCE_FILESYSTEM=1 \
@@ -89,3 +94,9 @@ PGLITE_EMSCRIPTEN_FLAGS="-sWASM_BIGINT \
 -sEXPORTED_RUNTIME_METHODS=$EXPORTED_RUNTIME_METHODS"
 # Building pglite itself needs to be the last step because of the PRELOAD_FILES parameter (a list of files and folders) need to be available.
 PGLITE_CFLAGS="$PGLITE_CFLAGS $PGLITE_EMSCRIPTEN_FLAGS" emmake make PORTNAME=emscripten -j -C src/backend/ install-pglite || { echo 'emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 51; }
+
+# Step 3.1.1 pgcrypto - special case
+cd ./pglite/ && ./build-pgcrypto.sh && cd ../
+PGLITE_WITH_PGCRYPTO=1 emmake make PORTNAME=emscripten -C contrib/ dist 
+# hack for pgcrypto. since we're linking lssl and lcrypto directly to the extension, their respective symbols should not be exported
+# find / -name pgcrypto.imports -exec rm -f {} \;

build-with-docker.sh

@@ -8,6 +8,6 @@ docker run $@ \
   --workdir=${DOCKER_WORKSPACE} \
   -v .:${DOCKER_WORKSPACE}:rw \
   -v ./dist:/install/pglite:rw \
-  electricsql/pglite-builder:3.1.74_2 \
+  electricsql/pglite-builder:3.1.74_4 \
   ./build-pglite.sh
 

contrib/Makefile

@@ -57,6 +57,10 @@ else
 ALWAYS_SUBDIRS += pgcrypto sslinfo
 endif
 
+ifeq ($(PGLITE_WITH_PGCRYPTO), 1)
+SUBDIRS += pgcrypto
+endif
+
 ifneq ($(with_uuid),no)
 SUBDIRS += uuid-ossp
 else

contrib/pgcrypto/Makefile

@@ -62,9 +62,6 @@ endif
 # shared library link.  (The order in which you list them here doesn't
 # matter.)
 SHLIB_LINK += $(filter -lcrypto -lz, $(LIBS))
-ifeq ($(PORTNAME), emscripten)
-SHLIB_LINK += -lcrypto
-endif
 ifeq ($(PORTNAME), win32)
 SHLIB_LINK += $(filter -leay32, $(LIBS))
 # those must be at the end

pglite-wasm/builder/Dockerfile

@@ -22,7 +22,7 @@ WORKDIR /install/libs
 WORKDIR /src
 
 # ENV EMCC_COMMON_FLAGS="-fPIC -sWASM_BIGINT -sMIN_SAFARI_VERSION=150000 -O2 -m32 -D_FILE_OFFSET_BITS=64 -sSUPPORT_LONGJMP=emscripten -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn"
-ENV EMCC_COMMON_FLAGS="-O2 -fPIC"
+ENV EMCC_COMMON_FLAGS="-O2 -fPIC -sWASM_BIGINT"
 
 WORKDIR /src
 RUN curl -L https://www.zlib.net/zlib-1.3.1.tar.gz | tar -xz
@@ -34,7 +34,7 @@ RUN ${LLVM_NM} /install/libs/lib/libz.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed
 WORKDIR /src
 RUN curl -L https://gitlab.gnome.org/GNOME/libxml2/-/archive/v2.14.5/libxml2-v2.14.5.tar.gz | tar -xz
 WORKDIR /src/libxml2-v2.14.5
-RUN ./autogen.sh --with-python=no
+RUN ./autogen.sh --with-python=no --with-threads=no
 RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./configure --enable-shared=no --enable-static=yes --with-python=no --prefix=/install/libs
 RUN emmake make -j && emmake make install
 # extract exported symbols - useful for passing them to emscripten as EXPORTED_FUNCTIONS 
@@ -52,11 +52,12 @@ RUN ${LLVM_NM} /install/libs/lib/libxslt.a | awk '$2 ~ /^[TDB]$/ {print $3}' | s
 WORKDIR /src
 RUN curl -L https://github.com/openssl/openssl/releases/download/openssl-3.0.17/openssl-3.0.17.tar.gz | tar xz
 WORKDIR /src/openssl-3.0.17
-RUN emconfigure ./Configure no-tests linux-generic64 --prefix=/install/libs
+RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./config no-sse2 no-hw no-asm no-tests no-threads no-shared linux-generic32 --prefix=/install/libs
 RUN sed -i 's|^CROSS_COMPILE.*$|CROSS_COMPILE=|g' Makefile # see https://github.com/emscripten-core/emscripten/issues/19597#issue-1754476454
-RUN emmake make -j && emmake make install
+RUN emmake make -j && emmake make install_sw install_ssldirs
 # extract exported symbols - useful for passing them to emscripten as EXPORTED_FUNCTIONS 
 RUN ${LLVM_NM} /install/libs/lib/libssl.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libssl.exports
+RUN ${LLVM_NM} /install/libs/lib/libcrypto.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libcrypto.exports
 
 WORKDIR /src
 RUN curl -L ftp://ftp.ossp.org/pkg/lib/uuid/uuid-1.6.2.tar.gz | tar xz

pglite-wasm/included.pglite.imports

@@ -1,33 +1,96 @@
+__errno_location
+appendStringInfo
+appendStringInfoChar
+appendStringInfoString
+atexit
+atoi
+BuildTupleFromCStrings
 close
+cstring_to_text
+cstring_to_text_with_len
+deconstruct_array_builtin
+downcase_truncate_identifier
+enlargeStringInfo
+errcode
 fcntl
+fopen
 free
+gai_strerror
+gen_random_uuid
+get_call_result_type
+GetDatabaseEncoding
+getegid
+getenv
+geteuid
+getgid
 getpid
 gettimeofday
+getuid
 gmtime
+initStringInfo
 interactive_one
 ioctl
 isalnum
 isxdigit
 lowerstr
 main
 malloc
+memchr
 memcmp
 memcpy
+memmove
+MemoryContextAlloc
+MemoryContextAllocZero
 memset
 nanosleep
 open
+palloc
+palloc0
+perror
+pfree
+pg_any_to_server
+pg_do_encoding_conversion
+pg_is_ascii
+pg_strcasecmp
+pg_strong_random
 pgl_backend
 pgl_initdb
 pgl_shutdown
+pstrdup
+puts
+qsort
 rand
 read
+readdir
 readstoplist
 realloc
+repalloc
+ResourceOwnerEnlarge
+ResourceOwnerForget
+ResourceOwnerRemember
 searchstoplist
 set_read_write_cbs
 socket
 srand
+sscanf
+stderr
+strcat
+strchr
 strcmp
+strcpy
+strcspn
+strdup
+strerror
+strerror_r
 strftime
+strlcpy
 strlen
-strtoul
+strncat
+strncmp
+strstr
+strtoul
+text_to_cstring
+text_to_cstring_buffer
+time
+tolower
+TupleDescGetAttInMetadata

pglite/Makefile

@@ -6,7 +6,8 @@ include $(top_builddir)/src/Makefile.global
 SUBDIRS = \
 		pg_ivm \
 		vector \
-		pgtap
+		pgtap \
+		pg_uuidv7
 
 prefix ?= /install/pglite
 EXTENSIONS_BUILD_ROOT := /tmp/extensions/build

pglite/build-pgcrypto.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+pushd ../contrib/pgcrypto
+# these flags are used in pgxs.mk (postgresql extension makefile) and passed to the build process of that extension
+emmake make LDFLAGS_SL="-sWASM_BIGINT -sSIDE_MODULE=1 -fexceptions -Wl,--whole-archive -lssl -lcrypto -Wl,--no-whole-archive" CFLAGS_SL="$PGLITE_CFLAGS -fexceptions -sWASM_BIGINT" -j
+# emmake make PORTNAME=emscripten dist
+popd

src/backend/Makefile

@@ -95,7 +95,7 @@ postgres: $(OBJS)
 # excludes the one in excluded.pglite.imports and adds a leading _ to each.
 pglite-exported-functions:
 	$(MKDIR_P) '$(emscripten_imports_dir)'
-	cat $(top_builddir)/pglite-wasm/excluded.*.imports $(top_builddir)/src/interfaces/libpq/exports.list $(LIB_EXPORTS_DIR)/libossp-uuid.exports | sort -u > '$(top_builddir)/pglite-wasm/excluded.imports'
+	cat $(top_builddir)/pglite-wasm/excluded.*.imports $(top_builddir)/src/interfaces/libpq/exports.list $(LIB_EXPORTS_DIR)/libossp-uuid.exports /install/exports/libcrypto.exports /install/exports/libssl.exports | sort -u > '$(top_builddir)/pglite-wasm/excluded.imports'
 	cat $(DESTDIR)$(emscripten_extension_imports_dir)/*.imports '$(top_builddir)/pglite-wasm/included.pglite.imports' | \
 	sort -u | \
 	grep -Fvx -f '$(top_builddir)/pglite-wasm/excluded.imports' | \
@@ -110,8 +110,8 @@ pglite: pglite-exported-functions
 	$(CC) $(CFLAGS) $(LDFLAGS) -DPG_PREFIX=/tmp/pglite -I$(top_builddir)/src/include -I$(top_builddir)/src/ -I$(top_builddir)/src/interfaces/libpq -o pglite.o -c $(top_builddir)/$(PGLITE_MAIN) -Wno-incompatible-pointer-types-discards-qualifiers
 	$(CC) \
 	$(PGLITE_CFLAGS) \
-	-fPIC -m32 -D_FILE_OFFSET_BITS=64 -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn \
-	-DPYDK=1 -DPG_PREFIX=/tmp/pglite -o pglite.html \
+	-fPIC -m32 -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn \
+	-DPG_PREFIX=/tmp/pglite -o pglite.html \
         $(PGPRELOAD) \
         -ferror-limit=1 \
 		-sEXPORTED_FUNCTIONS=@$(emscripten_imports_dir)/exported_functions.txt \

src/include/port/pg_pthread.h

@@ -12,9 +12,6 @@
 
 #ifndef PG_PTHREAD_H
 #define PG_PTHREAD_H
-#if defined(__wasi__)
-#define PYDK
-#endif /* __wasi__ */
 #include <pthread.h>
 
 #ifndef HAVE_PTHREAD_BARRIER_WAIT