diff --git a/.gitignore b/.gitignore index 682db69d4d24c..f9d8874941d77 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,4 @@ pglite.js pglite.html *.map pglite-wasm/excluded.imports +*.symbols \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index 94158709d174c..60cfbe40928c7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "pglite/pgtap"] path = pglite/pgtap url = https://github.com/theory/pgtap.git +[submodule "pglite/pg_uuidv7"] + path = pglite/pg_uuidv7 + url = https://github.com/fboulnois/pg_uuidv7/ diff --git a/build-pglite.sh b/build-pglite.sh index 3500a9dc1ebee..9e71fa772e4fe 100755 --- a/build-pglite.sh +++ b/build-pglite.sh @@ -8,13 +8,15 @@ INSTALL_FOLDER=${INSTALL_FOLDER:-"/install/pglite"} # build with optimizations by default aka release -PGLITE_CFLAGS="-O2" +# PGLITE_CFLAGS="-sDYLINK_DEBUG=2 -g -gsource-map --no-wasm-opt -Wbad-function-cast -Wcast-function-type" +PGLITE_CFLAGS="" if [ "$DEBUG" = true ] then echo "pglite: building debug version." - PGLITE_CFLAGS="-g -gsource-map --no-wasm-opt" + PGLITE_CFLAGS="$PGLITE_CFLAGS -g -gsource-map --no-wasm-opt --emit-symbol-map" else echo "pglite: building release version." + PGLITE_CFLAGS="$PGLITE_CFLAGS -O2" # we shouldn't need to do this, but there's a bug somewhere that prevents a successful build if this is set unset DEBUG fi @@ -38,7 +40,7 @@ else fi # we define here "all" emscripten flags in order to allow native builds (like libpglite) -EXPORTED_RUNTIME_METHODS="addFunction,removeFunction,FS,MEMFS" +EXPORTED_RUNTIME_METHODS="addFunction,removeFunction,FS,MEMFS,wasmTable" PGLITE_EMSCRIPTEN_FLAGS="-sWASM_BIGINT \ -sSUPPORT_LONGJMP=emscripten \ -sFORCE_FILESYSTEM=1 \ @@ -65,11 +67,14 @@ emmake make PORTNAME=emscripten -j || { echo 'error: emmake make PORTNAME=emscri emmake make PORTNAME=emscripten install || { echo 'error: emmake make PORTNAME=emscripten install' ; exit 22; } # Step 3.1: make all contrib extensions - do not install + +# Step 3.1.2 all the rest of contrib emmake make PORTNAME=emscripten -C contrib/ -j || { echo 'error: emmake make PORTNAME=emscripten -C contrib/ -j' ; exit 31; } # Step 3.2: make dist contrib extensions - this will create an archive for each extension emmake make PORTNAME=emscripten -C contrib/ dist || { echo 'error: emmake make PORTNAME=emscripten -C contrib/ dist' ; exit 32; } # the above will also create a file with the imports that each extension needs - we pass these as input in the next step for emscripten to keep alive + # Step 4: make and dist other extensions SAVE_PATH=$PATH PATH=$PATH:$INSTALL_FOLDER/bin @@ -78,7 +83,7 @@ emmake make OPTFLAGS="" PORTNAME=emscripten -C pglite/ dist || { echo 'error: ma PATH=$SAVE_PATH # Step 5: make and install pglite -EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack,addFunction,removeFunction" +EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack,addFunction,removeFunction,wasmTable" PGLITE_EMSCRIPTEN_FLAGS="-sWASM_BIGINT \ -sSUPPORT_LONGJMP=emscripten \ -sFORCE_FILESYSTEM=1 \ @@ -89,3 +94,9 @@ PGLITE_EMSCRIPTEN_FLAGS="-sWASM_BIGINT \ -sEXPORTED_RUNTIME_METHODS=$EXPORTED_RUNTIME_METHODS" # Building pglite itself needs to be the last step because of the PRELOAD_FILES parameter (a list of files and folders) need to be available. PGLITE_CFLAGS="$PGLITE_CFLAGS $PGLITE_EMSCRIPTEN_FLAGS" emmake make PORTNAME=emscripten -j -C src/backend/ install-pglite || { echo 'emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 51; } + +# Step 3.1.1 pgcrypto - special case +cd ./pglite/ && ./build-pgcrypto.sh && cd ../ +PGLITE_WITH_PGCRYPTO=1 emmake make PORTNAME=emscripten -C contrib/ dist +# hack for pgcrypto. since we're linking lssl and lcrypto directly to the extension, their respective symbols should not be exported +# find / -name pgcrypto.imports -exec rm -f {} \; \ No newline at end of file diff --git a/build-with-docker.sh b/build-with-docker.sh index 6a849b52ab863..2e7bbddbe24a3 100755 --- a/build-with-docker.sh +++ b/build-with-docker.sh @@ -8,6 +8,6 @@ docker run $@ \ --workdir=${DOCKER_WORKSPACE} \ -v .:${DOCKER_WORKSPACE}:rw \ -v ./dist:/install/pglite:rw \ - electricsql/pglite-builder:3.1.74_2 \ + electricsql/pglite-builder:3.1.74_4 \ ./build-pglite.sh diff --git a/contrib/Makefile b/contrib/Makefile index 603636cd4d248..ba794bfc35d60 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -57,6 +57,10 @@ else ALWAYS_SUBDIRS += pgcrypto sslinfo endif +ifeq ($(PGLITE_WITH_PGCRYPTO), 1) +SUBDIRS += pgcrypto +endif + ifneq ($(with_uuid),no) SUBDIRS += uuid-ossp else diff --git a/contrib/pgcrypto/Makefile b/contrib/pgcrypto/Makefile index 8085114dca6c2..5efa10c334c80 100644 --- a/contrib/pgcrypto/Makefile +++ b/contrib/pgcrypto/Makefile @@ -62,9 +62,6 @@ endif # shared library link. (The order in which you list them here doesn't # matter.) SHLIB_LINK += $(filter -lcrypto -lz, $(LIBS)) -ifeq ($(PORTNAME), emscripten) -SHLIB_LINK += -lcrypto -endif ifeq ($(PORTNAME), win32) SHLIB_LINK += $(filter -leay32, $(LIBS)) # those must be at the end diff --git a/pglite-wasm/builder/Dockerfile b/pglite-wasm/builder/Dockerfile index cf613dd2ddc85..8bcf3c6bb6aa3 100644 --- a/pglite-wasm/builder/Dockerfile +++ b/pglite-wasm/builder/Dockerfile @@ -22,7 +22,7 @@ WORKDIR /install/libs WORKDIR /src # ENV EMCC_COMMON_FLAGS="-fPIC -sWASM_BIGINT -sMIN_SAFARI_VERSION=150000 -O2 -m32 -D_FILE_OFFSET_BITS=64 -sSUPPORT_LONGJMP=emscripten -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn" -ENV EMCC_COMMON_FLAGS="-O2 -fPIC" +ENV EMCC_COMMON_FLAGS="-O2 -fPIC -sWASM_BIGINT" WORKDIR /src RUN curl -L https://www.zlib.net/zlib-1.3.1.tar.gz | tar -xz @@ -34,7 +34,7 @@ RUN ${LLVM_NM} /install/libs/lib/libz.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed WORKDIR /src RUN curl -L https://gitlab.gnome.org/GNOME/libxml2/-/archive/v2.14.5/libxml2-v2.14.5.tar.gz | tar -xz WORKDIR /src/libxml2-v2.14.5 -RUN ./autogen.sh --with-python=no +RUN ./autogen.sh --with-python=no --with-threads=no RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./configure --enable-shared=no --enable-static=yes --with-python=no --prefix=/install/libs RUN emmake make -j && emmake make install # extract exported symbols - useful for passing them to emscripten as EXPORTED_FUNCTIONS @@ -52,11 +52,12 @@ RUN ${LLVM_NM} /install/libs/lib/libxslt.a | awk '$2 ~ /^[TDB]$/ {print $3}' | s WORKDIR /src RUN curl -L https://github.com/openssl/openssl/releases/download/openssl-3.0.17/openssl-3.0.17.tar.gz | tar xz WORKDIR /src/openssl-3.0.17 -RUN emconfigure ./Configure no-tests linux-generic64 --prefix=/install/libs +RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./config no-sse2 no-hw no-asm no-tests no-threads no-shared linux-generic32 --prefix=/install/libs RUN sed -i 's|^CROSS_COMPILE.*$|CROSS_COMPILE=|g' Makefile # see https://github.com/emscripten-core/emscripten/issues/19597#issue-1754476454 -RUN emmake make -j && emmake make install +RUN emmake make -j && emmake make install_sw install_ssldirs # extract exported symbols - useful for passing them to emscripten as EXPORTED_FUNCTIONS RUN ${LLVM_NM} /install/libs/lib/libssl.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libssl.exports +RUN ${LLVM_NM} /install/libs/lib/libcrypto.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libcrypto.exports WORKDIR /src RUN curl -L ftp://ftp.ossp.org/pkg/lib/uuid/uuid-1.6.2.tar.gz | tar xz diff --git a/pglite-wasm/included.pglite.imports b/pglite-wasm/included.pglite.imports index 7a5d30d895111..ad348a8178828 100644 --- a/pglite-wasm/included.pglite.imports +++ b/pglite-wasm/included.pglite.imports @@ -1,9 +1,33 @@ +__errno_location +appendStringInfo +appendStringInfoChar +appendStringInfoString +atexit +atoi +BuildTupleFromCStrings close +cstring_to_text +cstring_to_text_with_len +deconstruct_array_builtin +downcase_truncate_identifier +enlargeStringInfo +errcode fcntl +fopen free +gai_strerror +gen_random_uuid +get_call_result_type +GetDatabaseEncoding +getegid +getenv +geteuid +getgid getpid gettimeofday +getuid gmtime +initStringInfo interactive_one ioctl isalnum @@ -11,23 +35,62 @@ isxdigit lowerstr main malloc +memchr memcmp memcpy +memmove +MemoryContextAlloc +MemoryContextAllocZero memset nanosleep open +palloc +palloc0 +perror +pfree +pg_any_to_server +pg_do_encoding_conversion +pg_is_ascii +pg_strcasecmp +pg_strong_random pgl_backend pgl_initdb pgl_shutdown +pstrdup +puts +qsort rand read +readdir readstoplist realloc +repalloc +ResourceOwnerEnlarge +ResourceOwnerForget +ResourceOwnerRemember searchstoplist set_read_write_cbs socket srand +sscanf +stderr +strcat +strchr strcmp +strcpy +strcspn +strdup +strerror +strerror_r strftime +strlcpy strlen -strtoul \ No newline at end of file +strncat +strncmp +strstr +strtoul +text_to_cstring +text_to_cstring_buffer +time +tolower +TupleDescGetAttInMetadata \ No newline at end of file diff --git a/pglite/Makefile b/pglite/Makefile index f6bc99b8e50cc..633fe686b9eba 100644 --- a/pglite/Makefile +++ b/pglite/Makefile @@ -6,7 +6,8 @@ include $(top_builddir)/src/Makefile.global SUBDIRS = \ pg_ivm \ vector \ - pgtap + pgtap \ + pg_uuidv7 prefix ?= /install/pglite EXTENSIONS_BUILD_ROOT := /tmp/extensions/build diff --git a/pglite/build-pgcrypto.sh b/pglite/build-pgcrypto.sh new file mode 100755 index 0000000000000..1d7369190fd8d --- /dev/null +++ b/pglite/build-pgcrypto.sh @@ -0,0 +1,6 @@ +#!/bin/bash +pushd ../contrib/pgcrypto +# these flags are used in pgxs.mk (postgresql extension makefile) and passed to the build process of that extension +emmake make LDFLAGS_SL="-sWASM_BIGINT -sSIDE_MODULE=1 -fexceptions -Wl,--whole-archive -lssl -lcrypto -Wl,--no-whole-archive" CFLAGS_SL="$PGLITE_CFLAGS -fexceptions -sWASM_BIGINT" -j +# emmake make PORTNAME=emscripten dist +popd \ No newline at end of file diff --git a/pglite/pg_uuidv7 b/pglite/pg_uuidv7 new file mode 160000 index 0000000000000..c707aae241118 --- /dev/null +++ b/pglite/pg_uuidv7 @@ -0,0 +1 @@ +Subproject commit c707aae2411181be4802f5fa565b44d9c0bcbc29 diff --git a/src/backend/Makefile b/src/backend/Makefile index 0bc6a84c3b10a..d4e40e4d410bb 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -95,7 +95,7 @@ postgres: $(OBJS) # excludes the one in excluded.pglite.imports and adds a leading _ to each. pglite-exported-functions: $(MKDIR_P) '$(emscripten_imports_dir)' - cat $(top_builddir)/pglite-wasm/excluded.*.imports $(top_builddir)/src/interfaces/libpq/exports.list $(LIB_EXPORTS_DIR)/libossp-uuid.exports | sort -u > '$(top_builddir)/pglite-wasm/excluded.imports' + cat $(top_builddir)/pglite-wasm/excluded.*.imports $(top_builddir)/src/interfaces/libpq/exports.list $(LIB_EXPORTS_DIR)/libossp-uuid.exports /install/exports/libcrypto.exports /install/exports/libssl.exports | sort -u > '$(top_builddir)/pglite-wasm/excluded.imports' cat $(DESTDIR)$(emscripten_extension_imports_dir)/*.imports '$(top_builddir)/pglite-wasm/included.pglite.imports' | \ sort -u | \ grep -Fvx -f '$(top_builddir)/pglite-wasm/excluded.imports' | \ @@ -110,8 +110,8 @@ pglite: pglite-exported-functions $(CC) $(CFLAGS) $(LDFLAGS) -DPG_PREFIX=/tmp/pglite -I$(top_builddir)/src/include -I$(top_builddir)/src/ -I$(top_builddir)/src/interfaces/libpq -o pglite.o -c $(top_builddir)/$(PGLITE_MAIN) -Wno-incompatible-pointer-types-discards-qualifiers $(CC) \ $(PGLITE_CFLAGS) \ - -fPIC -m32 -D_FILE_OFFSET_BITS=64 -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn \ - -DPYDK=1 -DPG_PREFIX=/tmp/pglite -o pglite.html \ + -fPIC -m32 -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn \ + -DPG_PREFIX=/tmp/pglite -o pglite.html \ $(PGPRELOAD) \ -ferror-limit=1 \ -sEXPORTED_FUNCTIONS=@$(emscripten_imports_dir)/exported_functions.txt \ diff --git a/src/include/port/pg_pthread.h b/src/include/port/pg_pthread.h index 6922eb423b945..b3d4ffbf59353 100644 --- a/src/include/port/pg_pthread.h +++ b/src/include/port/pg_pthread.h @@ -12,9 +12,6 @@ #ifndef PG_PTHREAD_H #define PG_PTHREAD_H -#if defined(__wasi__) -#define PYDK -#endif /* __wasi__ */ #include #ifndef HAVE_PTHREAD_BARRIER_WAIT