From 8d48e9788ac594eeeda9a7715b52aa0f2dfa5898 Mon Sep 17 00:00:00 2001 From: Chapman Flack Date: Mon, 17 Feb 2025 20:36:23 -0500 Subject: [PATCH 1/5] Use pgjdbc rather than pgjdbc-ng in CI JEP 472 issues warnings (one day to be errors) for uses of System.loadLibrary or when native methods are first bound according to the default binding process. PL/Java's use of JNI escapes without warnings because the JVM is launched from C and uses RegisterNatives to bind the native methods explicitly, so it does not involve loadLibrary or any use of the default native method matching. On the other hand, the pgjdbc-ng library furnishes netty, which has a use of loadLibrary, and produces warnings from the client side in the CI script. As Node.java now supports both pgjdbc and pgjdbc-ng, switch to pgjdbc in the CI scripts to avoid a nuisance warning. --- .github/workflows/ci-runnerpg.yml | 9 ++++----- .travis.yml | 7 +++---- appveyor.yml | 5 ++--- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci-runnerpg.yml b/.github/workflows/ci-runnerpg.yml index 9eea358cd..b43d38f70 100644 --- a/.github/workflows/ci-runnerpg.yml +++ b/.github/workflows/ci-runnerpg.yml @@ -79,7 +79,7 @@ jobs: working-directory: pljava run: | mvn clean install --batch-mode \ - -Psaxon-examples -Ppgjdbc-ng \ + -Psaxon-examples -Ppgjdbc \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn - name: Build PL/Java (Windows MinGW-w64) @@ -127,7 +127,7 @@ jobs: HOME=$( (cd .. && pwd) ) "$M2"/mvn clean install --batch-mode \ -Dpgsql.pgconfig="$PGBIN"'\pg_config' \ - -Psaxon-examples -Ppgjdbc-ng \ + -Psaxon-examples -Ppgjdbc \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn - name: Install and test PL/Java @@ -151,8 +151,8 @@ jobs: saxonVer=${saxonVer##*/} jdbcJar=$( - find "$mavenRepo/com/impossibl/pgjdbc-ng/pgjdbc-ng-all" \ - -name 'pgjdbc-ng-all-*.jar' -print | + find "$mavenRepo/org/postgresql/postgresql" \ + -name 'postgresql-*.jar' -print | sort | tail -n 1 ) @@ -200,7 +200,6 @@ jobs: "--class-path=$packageJar" \ "-J--add-modules=java.sql.rowset,jdk.httpserver" \ "-J-Dpgconfig=$pgConfig" \ - "-J-Dcom.impossibl.shadow.io.netty.noUnsafe=true" \ "-J-DmavenRepo=$mavenRepo" \ "-J-DsaxonVer=$saxonVer" - <<\ENDJSHELL diff --git a/.travis.yml b/.travis.yml index 3f3d9f36b..a0f7db6fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -119,7 +119,7 @@ install: | "$mvn" clean install --batch-mode \ -Dpgsql.pgconfig="$pgConfig" \ -Dpljava.libjvmdefault="$libjvm" \ - -Psaxon-examples -Ppgjdbc-ng \ + -Psaxon-examples -Ppgjdbc \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn script: | @@ -136,8 +136,8 @@ script: | saxonVer=${saxonVer##*/} jdbcJar=$( - find "$mavenRepo/com/impossibl/pgjdbc-ng/pgjdbc-ng-all" \ - -name 'pgjdbc-ng-all-*.jar' -print | + find "$mavenRepo/org/postgresql/postgresql" \ + -name 'postgresql-*.jar' -print | sort | tail -n 1 ) @@ -150,7 +150,6 @@ script: | "--class-path=$packageJar" \ "-J--add-modules=java.sql.rowset" \ "-J-Dpgconfig=$pgConfig" \ - "-J-Dcom.impossibl.shadow.io.netty.noUnsafe=true" \ "-J-DmavenRepo=$mavenRepo" \ "-J-DsaxonVer=$saxonVer" - <<\ENDJSHELL && # continues after here document diff --git a/appveyor.yml b/appveyor.yml index d5edb9c53..f60825ff1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -65,8 +65,8 @@ test_script: Select-Object -Last 1 ).Directory.Name - $jdbcJar = (Join-Path $mavenRepo "com\impossibl\pgjdbc-ng\pgjdbc-ng-all" | - Get-ChildItem -Recurse -Filter pgjdbc-ng-all-*.jar | + $jdbcJar = (Join-Path $mavenRepo "org\postgresql\postgresql" | + Get-ChildItem -Recurse -Filter postgresql-*.jar | Select-Object -Last 1 ).FullName @@ -577,7 +577,6 @@ test_script: "-J--class-path=$packageJar;$jdbcJar" ` "--class-path=$packageJar" ` "-J--add-modules=java.sql.rowset,jdk.httpserver" ` - "-J-Dcom.impossibl.shadow.io.netty.noUnsafe=true" ` "-J-Dpgconfig=$pgConfig" ` "-J-DmavenRepo=$mavenRepo" ` "-J-DsaxonVer=$saxonVer" - From 0ff417f3ea550192f46a5b7e6c91aa72c6d6910b Mon Sep 17 00:00:00 2001 From: Chapman Flack Date: Mon, 17 Feb 2025 19:36:25 -0500 Subject: [PATCH 2/5] Factor out and unify the CI jshell script 8a6b1b4 noted that the nearly-identical blobs of jshell in three different CI service configs would be ripe for factoring out into one common version, but left that chore for another day. This may as well be the day. --- .github/workflows/ci-runnerpg.yml | 517 +------------------------- .travis.yml | 347 +---------------- CI/integration | 595 ++++++++++++++++++++++++++++++ appveyor.yml | 507 +------------------------ 4 files changed, 604 insertions(+), 1362 deletions(-) create mode 100644 CI/integration diff --git a/.github/workflows/ci-runnerpg.yml b/.github/workflows/ci-runnerpg.yml index b43d38f70..9634ba52c 100644 --- a/.github/workflows/ci-runnerpg.yml +++ b/.github/workflows/ci-runnerpg.yml @@ -201,518 +201,5 @@ jobs: "-J--add-modules=java.sql.rowset,jdk.httpserver" \ "-J-Dpgconfig=$pgConfig" \ "-J-DmavenRepo=$mavenRepo" \ - "-J-DsaxonVer=$saxonVer" - <<\ENDJSHELL - - boolean succeeding = false; // begin pessimistic - - import static java.nio.file.Files.createTempFile; - import static java.nio.file.Files.write; - import java.nio.file.Path; - import static java.nio.file.Paths.get; - import java.sql.Connection; - import java.sql.PreparedStatement; - import java.sql.ResultSet; - import org.postgresql.pljava.packaging.Node; - import static org.postgresql.pljava.packaging.Node.q; - import static org.postgresql.pljava.packaging.Node.stateMachine; - import static org.postgresql.pljava.packaging.Node.isVoidResultSet; - import static org.postgresql.pljava.packaging.Node.s_isWindows; - import static - org.postgresql.pljava.packaging.Node.NOTHING_OR_PGJDBC_ZERO_COUNT; - /* - * Imports that will be needed to serve a jar file over http - * when the time comes for testing that. - */ - import static java.nio.charset.StandardCharsets.UTF_8; - import java.util.jar.Attributes; - import java.util.jar.Manifest; - import java.util.jar.JarOutputStream; - import java.util.zip.ZipEntry; - import com.sun.net.httpserver.BasicAuthenticator; - import com.sun.net.httpserver.HttpContext; - import com.sun.net.httpserver.HttpExchange; - import com.sun.net.httpserver.HttpHandler; - import com.sun.net.httpserver.HttpServer; - - String javaHome = System.getProperty("java.home"); - - Path javaLibDir = get(javaHome, s_isWindows ? "bin" : "lib"); - - Path libjvm = ( - "Mac OS X".equals(System.getProperty("os.name")) - ? Stream.of("libjli.dylib", "jli/libjli.dylib") - .map(s -> javaLibDir.resolve(s)) - .filter(Files::exists).findFirst().get() - : javaLibDir.resolve(s_isWindows ? "jvm.dll" : "server/libjvm.so") - ); - - String vmopts = - "-enableassertions:org.postgresql.pljava... -Xcheck:jni"; - - if ( 17 < Runtime.version().feature() ) - vmopts += " -Djava.security.manager=allow"; - - Node n1 = Node.get_new_node("TestNode1"); - - if ( s_isWindows ) - n1.use_pg_ctl(true); - - /* - * Keep a tally of the three types of diagnostic notices that may be - * received, and, independently, how many represent no-good test results - * (error always, but also warning if seen from the tests in the - * examples.jar deployment descriptor). - */ - Map results = - Stream.of("info", "warning", "error", "ng").collect( - LinkedHashMap::new, - (m,k) -> m.put(k, 0), (r,s) -> {}); - - boolean isDiagnostic(Object o, Set whatIsNG) - { - if ( ! ( o instanceof Throwable ) ) - return false; - String[] parts = Node.classify((Throwable)o); - String type = parts[0]; - String message = parts[2]; - results.compute(type, (k,v) -> 1 + v); - if ( whatIsNG.contains(type) ) - if ( ! "warning".equals(type) - || ! message.startsWith("[JEP 411]") ) - results.compute("ng", (k,v) -> 1 + v); - return true; - } - - /* - * Write a trial policy into a temporary file in n's data_dir, - * and set pljava.vmoptions accordingly over connection c. - * Returns the 'succeeding' flag from the state machine looking - * at the command results. - */ - boolean useTrialPolicy(Node n, Connection c, List contents) - throws Exception - { - Path trialPolicy = - createTempFile(n.data_dir().getParent(), "trial", "policy"); - - write(trialPolicy, contents); - - PreparedStatement setVmOpts = c.prepareStatement( - "SELECT null::pg_catalog.void" + - " FROM pg_catalog.set_config('pljava.vmoptions', ?, false)" - ); - - setVmOpts.setString(1, vmopts + - " -Dorg.postgresql.pljava.policy.trial=" + trialPolicy.toUri()); - - return stateMachine( - "change pljava.vmoptions", - null, - - q(setVmOpts, setVmOpts::execute) - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, - (o,p,q) -> null == o - ); - } - - try ( - AutoCloseable t1 = n1.initialized_cluster(); - AutoCloseable t2 = n1.started_server(Map.of( - "client_min_messages", "info", - "pljava.vmoptions", vmopts, - "pljava.libjvm_location", libjvm.toString() - )); - ) - { - try ( Connection c = n1.connect() ) - { - succeeding = true; // become optimistic, will be using &= below - - succeeding &= stateMachine( - "create extension no result", - null, - - q(c, "CREATE EXTENSION pljava") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - // state 1: consume any diagnostics, or to state 2 with same item - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - - NOTHING_OR_PGJDBC_ZERO_COUNT, // state 2 - - // state 3: must be end of input - (o,p,q) -> null == o - ); - } - - /* - * Get a new connection; 'create extension' always sets a near-silent - * logging level, and PL/Java only checks once at VM start time, so in - * the same session where 'create extension' was done, logging is - * somewhat suppressed. - */ - try ( Connection c = n1.connect() ) - { - succeeding &= stateMachine( - "saxon path examples path", - null, - - Node.installSaxonAndExamplesAndPath(c, - System.getProperty("mavenRepo"), - System.getProperty("saxonVer"), - true) - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - // states 1,2: diagnostics* then a void result set (saxon install) - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, - - // states 3,4: diagnostics* then a void result set (set classpath) - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 3 : -4, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 5 : false, - - // states 5,6: diagnostics* then void result set (example install) - (o,p,q) -> isDiagnostic(o, Set.of("error", "warning")) ? 5 : -6, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 7 : false, - - // states 7,8: diagnostics* then a void result set (set classpath) - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 7 : -8, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 9 : false, - - // state 9: must be end of input - (o,p,q) -> null == o - ); - - /* - * Exercise TrialPolicy some. Need another connection to change - * vmoptions. Uses some example functions, so insert here before the - * test of undeploying the examples. - */ - try ( Connection c2 = n1.connect() ) - { - succeeding &= useTrialPolicy(n1, c2, List.of( - "grant {", - " permission", - " org.postgresql.pljava.policy.TrialPolicy$Permission;", - "};" - )); - - PreparedStatement tryForbiddenRead = c2.prepareStatement( - "SELECT" + - " CASE WHEN javatest.java_getsystemproperty('java.home')" + - " OPERATOR(pg_catalog.=) ?" + - " THEN javatest.logmessage('INFO', 'trial policy test ok')" + - " ELSE javatest.logmessage('WARNING', 'trial policy test ng')" + - " END" - ); - - tryForbiddenRead.setString(1, javaHome); - - succeeding &= stateMachine( - "try to read a forbidden property", - null, - - q(tryForbiddenRead, tryForbiddenRead::execute) - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error", "warning")) ? 1 : -2, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, - (o,p,q) -> null == o - ); - // done with connection c2 - } - - /* - * Spin up an http server with a little jar file to serve, and test - * that install_jar works with an http: url. - * - * First make a little jar empty but for a deployment descriptor. - */ - String ddrName = "foo.ddr"; - Attributes a = new Attributes(); - a.putValue("SQLJDeploymentDescriptor", "TRUE"); - Manifest m = new Manifest(); - m.getEntries().put(ddrName, a); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - JarOutputStream jos = new JarOutputStream(baos, m); - jos.putNextEntry(new ZipEntry(ddrName)); - jos.write( - ( - "SQLActions[]={\n\"BEGIN INSTALL\n" + - "SELECT javatest.logmessage('INFO'," + - " 'jar installed from http');\n" + - "END INSTALL\",\n\"BEGIN REMOVE\n" + - "BEGIN dummy\n" + - "END dummy;\n" + - "END REMOVE\"\n}\n" - ).getBytes(UTF_8) - ); - jos.closeEntry(); - jos.close(); - byte[] jar = baos.toByteArray(); - - /* - * Now an http server. - */ - HttpServer hs = - HttpServer.create(new InetSocketAddress("localhost", 0), 0); - - try ( - Connection c2 = n1.connect(); - AutoCloseable t = ((Supplier)() -> - { - hs.start(); - return () -> hs.stop(0); - } - ).get() - ) - { - InetSocketAddress addr = hs.getAddress(); - - String id = "bar", pw = "baz"; - - URL u = new URI( - "http", id+':'+pw, addr.getHostString(), addr.getPort(), - "/foo.jar", null, null - ).toURL(); - - HttpContext hc = hs.createContext( - u.getPath(), - new HttpHandler() - { - @Override - public void handle(HttpExchange t) throws IOException - { - try ( InputStream is = t.getRequestBody() ) { - is.readAllBytes(); - } - t.getResponseHeaders().add( - "Content-Type", "application/java-archive"); - t.sendResponseHeaders(200, jar.length); - try ( OutputStream os = t.getResponseBody() ) { - os.write(jar); - } - } - } - ); - - hc.setAuthenticator( - new BasicAuthenticator("CI realm") - // ("CI realm", UTF_8) only available in Java 14 or later - { - @Override - public boolean checkCredentials(String c_id, String c_pw) - { - return id.equals(c_id) && pw.equals(c_pw); - } - } - ); - - succeeding &= useTrialPolicy(n1, c2, List.of( - "grant codebase \"${org.postgresql.pljava.codesource}\" {", - " permission", - " java.net.URLPermission \"http:*\", \"GET:Accept\";", - "};" - )); - - succeeding &= stateMachine( - "install a jar over http", - null, - - Node.installJar(c2, u.toString(), "foo", true) - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error", "warning")) ? 1 : -2, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, - (o,p,q) -> null == o - ); - - // done with connection c2 again, and the http server - } - - /* - * Also confirm that the generated undeploy actions work. - */ - succeeding &= stateMachine( - "remove jar void result", - null, - - q(c, "SELECT sqlj.remove_jar('examples', true)") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, - (o,p,q) -> null == o - ); - - /* - * Get another new connection and make sure the extension can be - * loaded in a non-superuser session. - */ - try ( Connection c2 = n1.connect() ) - { - succeeding &= stateMachine( - "become non-superuser", - null, - - q(c2, - "CREATE ROLE alice;" + - "GRANT USAGE ON SCHEMA sqlj TO alice;" + - "SET SESSION AUTHORIZATION alice") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - NOTHING_OR_PGJDBC_ZERO_COUNT, - NOTHING_OR_PGJDBC_ZERO_COUNT, - NOTHING_OR_PGJDBC_ZERO_COUNT, - (o,p,q) -> null == o - ); - - succeeding &= stateMachine( - "load as non-superuser", - null, - - q(c2, "SELECT null::pg_catalog.void" + - " FROM sqlj.get_classpath('public')") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, - (o,p,q) -> null == o - ); - // done with connection c2 again - } - - /* - * Make sure the extension drops cleanly and nothing - * is left in sqlj. - */ - succeeding &= stateMachine( - "drop extension and schema no result", - null, - - q(c, "DROP EXTENSION pljava;DROP SCHEMA sqlj") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - NOTHING_OR_PGJDBC_ZERO_COUNT, - NOTHING_OR_PGJDBC_ZERO_COUNT, - (o,p,q) -> null == o - ); - } - - /* - * Get another new connection and confirm that the old, pre-extension, - * LOAD method of installing PL/Java works. It is largely obsolete in - * the era of extensions, but still covers the use case of installing - * PL/Java without admin access on the server filesystem to where - * CREATE EXTENSION requires the files to be; they can still be - * installed in some other writable location the server can read, and - * pljava.module_path set to the right locations of the jars, and the - * correct shared-object path given to LOAD. - * - * Also test the after-the-fact packaging up with CREATE EXTENSION - * FROM unpackaged. That officially goes away in PG 13, where the - * equivalent sequence - * CREATE EXTENSION pljava VERSION unpackaged - * \c - * ALTER EXTENSION pljava UPDATE - * should be tested instead. - */ - try ( Connection c = n1.connect() ) - { - int majorVersion = c.getMetaData().getDatabaseMajorVersion(); - - succeeding &= stateMachine( - "load as non-extension", - null, - - Node.loadPLJava(c) - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - NOTHING_OR_PGJDBC_ZERO_COUNT, - (o,p,q) -> null == o - ); - - if ( 13 <= majorVersion ) - { - succeeding &= stateMachine( - "create unpackaged (PG >= 13)", - null, - - q(c, "CREATE EXTENSION pljava VERSION unpackaged") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - NOTHING_OR_PGJDBC_ZERO_COUNT, - (o,p,q) -> null == o - ); - } - } - - /* - * CREATE EXTENSION FROM unpackaged (or the second half of the - * PG >= 13 CREATE EXTENSION VERSION unpackaged;ALTER EXTENSION UPDATE - * sequence) has to happen over a new connection. - */ - try ( Connection c = n1.connect() ) - { - int majorVersion = c.getMetaData().getDatabaseMajorVersion(); - - succeeding &= stateMachine( - "package after loading", - null, - - q(c, 13 > majorVersion - ? "CREATE EXTENSION pljava FROM unpackaged" - : "ALTER EXTENSION pljava UPDATE") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - NOTHING_OR_PGJDBC_ZERO_COUNT, - (o,p,q) -> null == o - ); - - /* - * Again make sure extension drops cleanly with nothing left behind. - */ - succeeding &= stateMachine( - "drop extension and schema no result", - null, - - q(c, "DROP EXTENSION pljava;DROP SCHEMA sqlj") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - NOTHING_OR_PGJDBC_ZERO_COUNT, - NOTHING_OR_PGJDBC_ZERO_COUNT, - (o,p,q) -> null == o - ); - } - } catch ( Throwable t ) - { - succeeding = false; - throw t; - } - - System.out.println(results); - succeeding &= (0 == results.get("ng")); - System.exit(succeeding ? 0 : 1); - ENDJSHELL + "-J-DsaxonVer=$saxonVer" \ + CI/integration diff --git a/.travis.yml b/.travis.yml index a0f7db6fe..9af39d682 100644 --- a/.travis.yml +++ b/.travis.yml @@ -151,347 +151,6 @@ script: | "-J--add-modules=java.sql.rowset" \ "-J-Dpgconfig=$pgConfig" \ "-J-DmavenRepo=$mavenRepo" \ - "-J-DsaxonVer=$saxonVer" - <<\ENDJSHELL && # continues after here document - - boolean succeeding = false; // begin pessimistic - - import static java.nio.file.Files.createTempFile - import static java.nio.file.Files.write - import java.nio.file.Path - import static java.nio.file.Paths.get - import java.sql.Connection - import java.sql.PreparedStatement - import java.sql.ResultSet - import org.postgresql.pljava.packaging.Node - import static org.postgresql.pljava.packaging.Node.q - import static org.postgresql.pljava.packaging.Node.stateMachine - import static org.postgresql.pljava.packaging.Node.isVoidResultSet - - String vmopts = "-enableassertions:org.postgresql.pljava... -Xcheck:jni" - - Node n1 = Node.get_new_node("TestNode1") - - /* - * Keep a tally of the three types of diagnostic notices that may be received, - * and, independently, how many represent no-good test results (error always, - * but also warning if seen from the tests in the examples.jar deployment - * descriptor). - */ - Map results = - Stream.of("info", "warning", "error", "ng").collect( - LinkedHashMap::new, (m,k) -> m.put(k, 0), (r,s) -> {}) - - boolean isDiagnostic(Object o, Set whatIsNG) - { - if ( ! ( o instanceof Throwable ) ) - return false; - String[] parts = Node.classify((Throwable)o); - String type = parts[0]; - String message = parts[2]; - results.compute(type, (k,v) -> 1 + v); - if ( whatIsNG.contains(type) ) - if ( ! "warning".equals(type) || ! message.startsWith("[JEP 411]") ) - results.compute("ng", (k,v) -> 1 + v); - return true; - } - - try ( - AutoCloseable t1 = n1.initialized_cluster(); - AutoCloseable t2 = n1.started_server(Map.of( - "client_min_messages", "info", - "pljava.vmoptions", vmopts - )); - ) - { - try ( Connection c = n1.connect() ) - { - succeeding = true; // become optimistic, will be using &= below - - succeeding &= stateMachine( - "create extension no result", - null, - - q(c, "CREATE EXTENSION pljava") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - // state 1: consume any diagnostics, or go to state 2 without consuming - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - - // state 2: must be end of input - (o,p,q) -> null == o - ); - } - - /* - * Get a new connection; 'create extension' always sets a near-silent - * logging level, and PL/Java only checks once at VM start time, so in - * the same session where 'create extension' was done, logging is - * somewhat suppressed. - */ - try ( Connection c = n1.connect() ) - { - succeeding &= stateMachine( - "saxon path examples path", - null, - - Node.installSaxonAndExamplesAndPath(c, - System.getProperty("mavenRepo"), - System.getProperty("saxonVer"), - true) - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - // states 1,2: maybe diagnostics, then a void result set (saxon install) - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, - - // states 3,4: maybe diagnostics, then a void result set (set classpath) - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 3 : -4, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 5 : false, - - // states 5,6: maybe diagnostics, then void result set (example install) - (o,p,q) -> isDiagnostic(o, Set.of("error", "warning")) ? 5 : -6, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 7 : false, - - // states 7,8: maybe diagnostics, then a void result set (set classpath) - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 7 : -8, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 9 : false, - - // state 9: must be end of input - (o,p,q) -> null == o - ); - - /* - * Exercise TrialPolicy some. Need another connection to change - * vmoptions. Uses some example functions, so insert here before the - * test of undeploying the examples. - */ - try ( Connection c2 = n1.connect() ) - { - Path trialPolicy = - createTempFile(n1.data_dir().getParent(), "trial", "policy"); - - write(trialPolicy, List.of( - "grant {", - " permission", - " org.postgresql.pljava.policy.TrialPolicy$Permission;", - "};" - )); - - PreparedStatement setVmOpts = c2.prepareStatement( - "SELECT null::pg_catalog.void" + - " FROM pg_catalog.set_config('pljava.vmoptions', ?, false)" - ); - - setVmOpts.setString(1, vmopts + - " -Dorg.postgresql.pljava.policy.trial=" + trialPolicy.toUri()); - - succeeding &= stateMachine( - "change pljava.vmoptions", - null, - - q(setVmOpts, setVmOpts::execute) - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, - (o,p,q) -> null == o - ); - - PreparedStatement tryForbiddenRead = c2.prepareStatement( - "SELECT" + - " CASE WHEN javatest.java_getsystemproperty('java.home')" + - " OPERATOR(pg_catalog.=) ?" + - " THEN javatest.logmessage('INFO', 'trial policy test ok')" + - " ELSE javatest.logmessage('WARNING', 'trial policy test ng')" + - " END" - ); - - tryForbiddenRead.setString(1, System.getProperty("java.home")); - - succeeding &= stateMachine( - "try to read a forbidden property", - null, - - q(tryForbiddenRead, tryForbiddenRead::execute) - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error", "warning")) ? 1 : -2, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, - (o,p,q) -> null == o - ); - // done with connection c2 - } - - /* - * Also confirm that the generated undeploy actions work. - */ - succeeding &= stateMachine( - "remove jar void result", - null, - - q(c, "SELECT sqlj.remove_jar('examples', true)") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, - (o,p,q) -> null == o - ); - - /* - * Get another new connection and make sure the extension can be - * loaded in a non-superuser session. - */ - try ( Connection c2 = n1.connect() ) - { - succeeding &= stateMachine( - "become non-superuser", - null, - - q(c2, - "CREATE ROLE alice;" + - "GRANT USAGE ON SCHEMA sqlj TO alice;" + - "SET SESSION AUTHORIZATION alice") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - (o,p,q) -> null == o - ); - - succeeding &= stateMachine( - "load as non-superuser", - null, - - q(c2, "SELECT null::pg_catalog.void" + - " FROM sqlj.get_classpath('public')") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, - (o,p,q) -> null == o - ); - // done with connection c2 again - } - - /* - * Make sure the extension drops cleanly and nothing - * is left in sqlj. - */ - succeeding &= stateMachine( - "drop extension and schema no result", - null, - - q(c, "DROP EXTENSION pljava;DROP SCHEMA sqlj") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - (o,p,q) -> null == o - ); - } - - /* - * Get another new connection and confirm that the old, pre-extension, - * LOAD method of installing PL/Java works. It is largely obsolete in - * the era of extensions, but still covers the use case of installing - * PL/Java without admin access on the server filesystem to where - * CREATE EXTENSION requires the files to be; they can still be - * installed in some other writable location the server can read, and - * pljava.module_path set to the right locations of the jars, and the - * correct shared-object path given to LOAD. - * - * Also test the after-the-fact packaging up with CREATE EXTENSION - * FROM unpackaged. That officially goes away in PG 13, where the - * equivalent sequence - * CREATE EXTENSION pljava VERSION unpackaged - * \c - * ALTER EXTENSION pljava UPDATE - * should be tested instead. - */ - try ( Connection c = n1.connect() ) - { - int majorVersion = c.getMetaData().getDatabaseMajorVersion(); - - succeeding &= stateMachine( - "load as non-extension", - null, - - Node.loadPLJava(c) - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - (o,p,q) -> null == o - ); - - if ( 13 <= majorVersion ) - { - succeeding &= stateMachine( - "create unpackaged (PG >= 13)", - null, - - q(c, "CREATE EXTENSION pljava VERSION unpackaged") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - (o,p,q) -> null == o - ); - } - } - - /* - * CREATE EXTENSION FROM unpackaged (or the second half of the - * PG >= 13 CREATE EXTENSION VERSION unpackaged;ALTER EXTENSION UPDATE - * sequence) has to happen over a new connection. - */ - try ( Connection c = n1.connect() ) - { - int majorVersion = c.getMetaData().getDatabaseMajorVersion(); - - succeeding &= stateMachine( - "package after loading", - null, - - q(c, 13 > majorVersion - ? "CREATE EXTENSION pljava FROM unpackaged" - : "ALTER EXTENSION pljava UPDATE") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - (o,p,q) -> null == o - ); - - /* - * Again make sure extension drops cleanly with nothing left behind. - */ - succeeding &= stateMachine( - "drop extension and schema no result", - null, - - q(c, "DROP EXTENSION pljava;DROP SCHEMA sqlj") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - (o,p,q) -> null == o - ); - } - } catch ( Throwable t ) - { - succeeding = false; - throw t; - } - - System.out.println(results); - succeeding &= (0 == results.get("ng")); - System.exit(succeeding ? 0 : 1) - ENDJSHELL - : travis wants something after the end of the here document + "-J-DsaxonVer=$saxonVer" \ + CI/integration + : travis wanted something here at the end once diff --git a/CI/integration b/CI/integration new file mode 100644 index 000000000..6f3caa65a --- /dev/null +++ b/CI/integration @@ -0,0 +1,595 @@ +/* + * Copyright (c) 2020-2025 Tada AB and other contributors, as listed below. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the The BSD 3-Clause License + * which accompanies this distribution, and is available at + * http://opensource.org/licenses/BSD-3-Clause + * + * Contributors: + * Chapman Flack + * Kartik Ohri + * + * This jshell script performs basic integration tests for PL/Java's CI. + * + * It must be executed with the built PL/Java packaged jar (produced by the + * pljava-packaging subproject) on the classpath, as well as a PGJDBC or + * pgjdbc-ng full jar. The PL/Java packaged jar includes a Node.class + * exporting functions not unlike the Perl module once called PostgresNode + * (and now called PostgreSQL::Test::Cluster) in the PostgreSQL distribution. + * The javadocs for Node.class explain the available functions. + * + * When jshell runs this script with -execution local, it needs both a + * --class-path and a -J--class-path argument. The former need only contain + * the PL/Java jar itself, so the contents are visible to jshell. The -J version + * passed to the underlying JVM needs both that jar and the PGJDBC or pgjdbc-ng + * driver jar. The driver classes need not be visible to jshell, but the JVM + * must be able to find them. + * + * Tests included in this script require + * -J--add-modules=java.sql.rowset,jdk.httpserver + * on the jshell command line. + * + * These Java properties must be set (as with -J-Dpgconfig=...) on the jshell + * command line: + * + * pgconfig + * the path to the pg_config executable that will be used to locate + * the PostgreSQL installation to be used in the tests + * mavenRepo + * the topmost directory of the local Maven repository. The Saxon jar + * downloaded as a dependency (when -Psaxon-examples was used on the mvn + * command line for building) will be found in this repository + * saxonVer + * the version of the Saxon library to use (appears in the library jar + * file name and as the name of its containing directory in the repository) + * + * These properties are optional (their absence is equivalent to a setting + * of false): + * + * redirectError + * if true, the standard error stream from the tests will be merged into + * the standard output stream. This can be desirable if this script is + * invoked from Windows PowerShell, which believes a standard error stream + * should only carry Error Records and makes an awful mess of anything else. + * extractFiles + * if true, begin by extracting and installing the PL/Java files from the jar + * into the proper locations indicated by the pg_config executable. If false, + * extraction will be skipped, assumed to have been done in a separate step + * simply running java -jar on the PL/Java packaged jar. Doing the extraction + * here can be useful, if this script is run with the needed permissions to + * write in the PostgreSQL install locations, when combined with redirectError + * if running under PowerShell, which would otherwise mess up the output. + * + * The script does not (yet) produce output in any standardized format such as + * TAP. The output will include numerous , , , or + * elements. If it runs to completion there will be a line with counts + * for info, warning, error, and ng. The count of ng results includes errors + * and certain warnings. The tests that are run from the deployment descriptor + * of the pljava-examples jar report test failures as warnings (to avoid cutting + * short the test as an error would), so those warnings are counted in ng. + * + * jshell will exit with a nonzero status if ng > 0 or anything else was seen + * to go wrong or the script did not run to completion. + */ +boolean succeeding = false; // begin pessimistic + +boolean redirectError = Boolean.getBoolean("redirectError"); + +if ( redirectError ) + System.setErr(System.out); // PowerShell makes a mess of stderr output + +UnaryOperator tweaks = + redirectError ? p -> p.redirectErrorStream(true) : UnaryOperator.identity(); + +import static java.nio.file.Files.createTempFile; +import static java.nio.file.Files.write; +import java.nio.file.Path; +import static java.nio.file.Paths.get; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import org.postgresql.pljava.packaging.Node; +import static org.postgresql.pljava.packaging.Node.q; +import static org.postgresql.pljava.packaging.Node.stateMachine; +import static org.postgresql.pljava.packaging.Node.isVoidResultSet; +import static org.postgresql.pljava.packaging.Node.s_isWindows; +import static + org.postgresql.pljava.packaging.Node.NOTHING_OR_PGJDBC_ZERO_COUNT; +/* + * Imports that will be needed to serve a jar file over http + * when the time comes for testing that. + */ +import static java.nio.charset.StandardCharsets.UTF_8; +import java.util.jar.Attributes; +import java.util.jar.Manifest; +import java.util.jar.JarOutputStream; +import java.util.zip.ZipEntry; +import com.sun.net.httpserver.BasicAuthenticator; +import com.sun.net.httpserver.HttpContext; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; + +if ( Boolean.getBoolean("extractFiles") ) + Node.main(new String[0]); // extract the files + +String javaHome = System.getProperty("java.home"); + +Path javaLibDir = get(javaHome, s_isWindows ? "bin" : "lib"); + +Path libjvm = ( + "Mac OS X".equals(System.getProperty("os.name")) + ? Stream.of("libjli.dylib", "jli/libjli.dylib") + .map(s -> javaLibDir.resolve(s)) + .filter(Files::exists).findFirst().get() + : javaLibDir.resolve(s_isWindows ? "jvm.dll" : "server/libjvm.so") +); + +String vmopts = "-enableassertions:org.postgresql.pljava... -Xcheck:jni"; + +if ( 17 < Runtime.version().feature() ) + vmopts += " -Djava.security.manager=allow"; + +Node n1 = Node.get_new_node("TestNode1"); + +if ( s_isWindows ) + n1.use_pg_ctl(true); + +/* + * Keep a tally of the three types of diagnostic notices that may be + * received, and, independently, how many represent no-good test results + * (error always, but also warning if seen from the tests in the + * examples.jar deployment descriptor). + */ +Map results = + Stream.of("info", "warning", "error", "ng").collect( + LinkedHashMap::new, + (m,k) -> m.put(k, 0), (r,s) -> {}); + +boolean isDiagnostic(Object o, Set whatIsNG) +{ + if ( ! ( o instanceof Throwable ) ) + return false; + String[] parts = Node.classify((Throwable)o); + String type = parts[0]; + String message = parts[2]; + results.compute(type, (k,v) -> 1 + v); + if ( whatIsNG.contains(type) ) + if ( ! "warning".equals(type) || ! message.startsWith("[JEP 411]") ) + results.compute("ng", (k,v) -> 1 + v); + return true; +} + +/* + * Write a trial policy into a temporary file in n's data_dir, + * and set pljava.vmoptions accordingly over connection c. + * Returns the 'succeeding' flag from the state machine looking + * at the command results. + */ +boolean useTrialPolicy(Node n, Connection c, List contents) +throws Exception +{ + Path trialPolicy = + createTempFile(n.data_dir().getParent(), "trial", "policy"); + + write(trialPolicy, contents); + + PreparedStatement setVmOpts = c.prepareStatement( + "SELECT null::pg_catalog.void" + + " FROM pg_catalog.set_config('pljava.vmoptions', ?, false)" + ); + + setVmOpts.setString(1, vmopts + + " -Dorg.postgresql.pljava.policy.trial=" + trialPolicy.toUri()); + + return stateMachine( + "change pljava.vmoptions", + null, + + q(setVmOpts, setVmOpts::execute) + .flatMap(Node::semiFlattenDiagnostics) + .peek(Node::peek), + + (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, + (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, + (o,p,q) -> null == o + ); +} + +try ( + AutoCloseable t1 = n1.initialized_cluster(tweaks); + AutoCloseable t2 = n1.started_server(Map.of( + "client_min_messages", "info", + "pljava.vmoptions", vmopts, + "pljava.libjvm_location", libjvm.toString() + ), tweaks); +) +{ + try ( Connection c = n1.connect() ) + { + succeeding = true; // become optimistic, will be using &= below + + succeeding &= stateMachine( + "create extension no result", + null, + + q(c, "CREATE EXTENSION pljava") + .flatMap(Node::semiFlattenDiagnostics) + .peek(Node::peek), + + // state 1: consume any diagnostics, or to state 2 with same item + (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, + + NOTHING_OR_PGJDBC_ZERO_COUNT, // state 2 + + // state 3: must be end of input + (o,p,q) -> null == o + ); + } + + /* + * Get a new connection; 'create extension' always sets a near-silent + * logging level, and PL/Java only checks once at VM start time, so in + * the same session where 'create extension' was done, logging is + * somewhat suppressed. + */ + try ( Connection c = n1.connect() ) + { + succeeding &= stateMachine( + "saxon path examples path", + null, + + Node.installSaxonAndExamplesAndPath(c, + System.getProperty("mavenRepo"), + System.getProperty("saxonVer"), + true) + .flatMap(Node::semiFlattenDiagnostics) + .peek(Node::peek), + + // states 1,2: diagnostics* then a void result set (saxon install) + (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, + (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, + + // states 3,4: diagnostics* then a void result set (set classpath) + (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 3 : -4, + (o,p,q) -> isVoidResultSet(o, 1, 1) ? 5 : false, + + // states 5,6: diagnostics* then void result set (example install) + (o,p,q) -> isDiagnostic(o, Set.of("error", "warning")) ? 5 : -6, + (o,p,q) -> isVoidResultSet(o, 1, 1) ? 7 : false, + + // states 7,8: diagnostics* then a void result set (set classpath) + (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 7 : -8, + (o,p,q) -> isVoidResultSet(o, 1, 1) ? 9 : false, + + // state 9: must be end of input + (o,p,q) -> null == o + ); + + /* + * Exercise TrialPolicy some. Need another connection to change + * vmoptions. Uses some example functions, so insert here before the + * test of undeploying the examples. + */ + try ( Connection c2 = n1.connect() ) + { + succeeding &= useTrialPolicy(n1, c2, List.of( + "grant {", + " permission", + " org.postgresql.pljava.policy.TrialPolicy$Permission;", + "};" + )); + + PreparedStatement tryForbiddenRead = c2.prepareStatement( + "SELECT" + + " CASE WHEN javatest.java_getsystemproperty('java.home')" + + " OPERATOR(pg_catalog.=) ?" + + " THEN javatest.logmessage('INFO', 'trial policy test ok')" + + " ELSE javatest.logmessage('WARNING', 'trial policy test ng')" + + " END" + ); + + tryForbiddenRead.setString(1, javaHome); + + succeeding &= stateMachine( + "try to read a forbidden property", + null, + + q(tryForbiddenRead, tryForbiddenRead::execute) + .flatMap(Node::semiFlattenDiagnostics) + .peek(Node::peek), + + (o,p,q) -> isDiagnostic(o, Set.of("error", "warning")) ? 1 : -2, + (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, + (o,p,q) -> null == o + ); + // done with connection c2 + } + + /* + * Spin up an http server with a little jar file to serve, and test + * that install_jar works with an http: url. + * + * First make a little jar empty but for a deployment descriptor. + */ + String ddrName = "foo.ddr"; + Attributes a = new Attributes(); + a.putValue("SQLJDeploymentDescriptor", "TRUE"); + Manifest m = new Manifest(); + m.getEntries().put(ddrName, a); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JarOutputStream jos = new JarOutputStream(baos, m); + jos.putNextEntry(new ZipEntry(ddrName)); + jos.write( + ( + "SQLActions[]={\n\"BEGIN INSTALL\n" + + "SELECT javatest.logmessage('INFO'," + + " 'jar installed from http');\n" + + "END INSTALL\",\n\"BEGIN REMOVE\n" + + "BEGIN dummy\n" + + "END dummy;\n" + + "END REMOVE\"\n}\n" + ).getBytes(UTF_8) + ); + jos.closeEntry(); + jos.close(); + byte[] jar = baos.toByteArray(); + + /* + * Now an http server. + */ + HttpServer hs = + HttpServer.create(new InetSocketAddress("localhost", 0), 0); + + try ( + Connection c2 = n1.connect(); + AutoCloseable t = ((Supplier)() -> + { + hs.start(); + return () -> hs.stop(0); + } + ).get() + ) + { + InetSocketAddress addr = hs.getAddress(); + + String id = "bar", pw = "baz"; + + URL u = new URI( + "http", id+':'+pw, addr.getHostString(), addr.getPort(), + "/foo.jar", null, null + ).toURL(); + + HttpContext hc = hs.createContext( + u.getPath(), + new HttpHandler() + { + @Override + public void handle(HttpExchange t) throws IOException + { + try ( InputStream is = t.getRequestBody() ) { + is.readAllBytes(); + } + t.getResponseHeaders().add( + "Content-Type", "application/java-archive"); + t.sendResponseHeaders(200, jar.length); + try ( OutputStream os = t.getResponseBody() ) { + os.write(jar); + } + } + } + ); + + hc.setAuthenticator( + new BasicAuthenticator("CI realm") + // ("CI realm", UTF_8) only available in Java 14 or later + { + @Override + public boolean checkCredentials(String c_id, String c_pw) + { + return id.equals(c_id) && pw.equals(c_pw); + } + } + ); + + succeeding &= useTrialPolicy(n1, c2, List.of( + "grant codebase \"${org.postgresql.pljava.codesource}\" {", + " permission", + " java.net.URLPermission \"http:*\", \"GET:Accept\";", + "};" + )); + + succeeding &= stateMachine( + "install a jar over http", + null, + + Node.installJar(c2, u.toString(), "foo", true) + .flatMap(Node::semiFlattenDiagnostics) + .peek(Node::peek), + + (o,p,q) -> isDiagnostic(o, Set.of("error", "warning")) ? 1 : -2, + (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, + (o,p,q) -> null == o + ); + + // done with connection c2 again, and the http server + } + + /* + * Also confirm that the generated undeploy actions work. + */ + succeeding &= stateMachine( + "remove jar void result", + null, + + q(c, "SELECT sqlj.remove_jar('examples', true)") + .flatMap(Node::semiFlattenDiagnostics) + .peek(Node::peek), + + (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, + (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, + (o,p,q) -> null == o + ); + + /* + * Get another new connection and make sure the extension can be + * loaded in a non-superuser session. + */ + try ( Connection c2 = n1.connect() ) + { + succeeding &= stateMachine( + "become non-superuser", + null, + + q(c2, + "CREATE ROLE alice;" + + "GRANT USAGE ON SCHEMA sqlj TO alice;" + + "SET SESSION AUTHORIZATION alice") + .flatMap(Node::semiFlattenDiagnostics) + .peek(Node::peek), + + (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, + NOTHING_OR_PGJDBC_ZERO_COUNT, + NOTHING_OR_PGJDBC_ZERO_COUNT, + NOTHING_OR_PGJDBC_ZERO_COUNT, + (o,p,q) -> null == o + ); + + succeeding &= stateMachine( + "load as non-superuser", + null, + + q(c2, "SELECT null::pg_catalog.void" + + " FROM sqlj.get_classpath('public')") + .flatMap(Node::semiFlattenDiagnostics) + .peek(Node::peek), + + (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, + (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, + (o,p,q) -> null == o + ); + // done with connection c2 again + } + + /* + * Make sure the extension drops cleanly and nothing + * is left in sqlj. + */ + succeeding &= stateMachine( + "drop extension and schema no result", + null, + + q(c, "DROP EXTENSION pljava;DROP SCHEMA sqlj") + .flatMap(Node::semiFlattenDiagnostics) + .peek(Node::peek), + + (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, + NOTHING_OR_PGJDBC_ZERO_COUNT, + NOTHING_OR_PGJDBC_ZERO_COUNT, + (o,p,q) -> null == o + ); + } + + /* + * Get another new connection and confirm that the old, pre-extension, + * LOAD method of installing PL/Java works. It is largely obsolete in + * the era of extensions, but still covers the use case of installing + * PL/Java without admin access on the server filesystem to where + * CREATE EXTENSION requires the files to be; they can still be + * installed in some other writable location the server can read, and + * pljava.module_path set to the right locations of the jars, and the + * correct shared-object path given to LOAD. + * + * Also test the after-the-fact packaging up with CREATE EXTENSION + * FROM unpackaged. That officially goes away in PG 13, where the + * equivalent sequence + * CREATE EXTENSION pljava VERSION unpackaged + * \c + * ALTER EXTENSION pljava UPDATE + * should be tested instead. + */ + try ( Connection c = n1.connect() ) + { + int majorVersion = c.getMetaData().getDatabaseMajorVersion(); + + succeeding &= stateMachine( + "load as non-extension", + null, + + Node.loadPLJava(c) + .flatMap(Node::semiFlattenDiagnostics) + .peek(Node::peek), + + (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, + NOTHING_OR_PGJDBC_ZERO_COUNT, + (o,p,q) -> null == o + ); + + if ( 13 <= majorVersion ) + { + succeeding &= stateMachine( + "create unpackaged (PG >= 13)", + null, + + q(c, "CREATE EXTENSION pljava VERSION unpackaged") + .flatMap(Node::semiFlattenDiagnostics) + .peek(Node::peek), + + (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, + NOTHING_OR_PGJDBC_ZERO_COUNT, + (o,p,q) -> null == o + ); + } + } + + /* + * CREATE EXTENSION FROM unpackaged (or the second half of the + * PG >= 13 CREATE EXTENSION VERSION unpackaged;ALTER EXTENSION UPDATE + * sequence) has to happen over a new connection. + */ + try ( Connection c = n1.connect() ) + { + int majorVersion = c.getMetaData().getDatabaseMajorVersion(); + + succeeding &= stateMachine( + "package after loading", + null, + + q(c, 13 > majorVersion + ? "CREATE EXTENSION pljava FROM unpackaged" + : "ALTER EXTENSION pljava UPDATE") + .flatMap(Node::semiFlattenDiagnostics) + .peek(Node::peek), + + (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, + NOTHING_OR_PGJDBC_ZERO_COUNT, + (o,p,q) -> null == o + ); + + /* + * Again make sure extension drops cleanly with nothing left behind. + */ + succeeding &= stateMachine( + "drop extension and schema no result", + null, + + q(c, "DROP EXTENSION pljava;DROP SCHEMA sqlj") + .flatMap(Node::semiFlattenDiagnostics) + .peek(Node::peek), + + (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, + NOTHING_OR_PGJDBC_ZERO_COUNT, + NOTHING_OR_PGJDBC_ZERO_COUNT, + (o,p,q) -> null == o + ); + } +} catch ( Throwable t ) +{ + succeeding = false; + throw t; +} + +System.out.println(results); +succeeding &= (0 == results.get("ng")); +System.exit(succeeding ? 0 : 1); diff --git a/appveyor.yml b/appveyor.yml index f60825ff1..acdea6078 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -70,508 +70,6 @@ test_script: Select-Object -Last 1 ).FullName - @' - boolean succeeding = false; // begin pessimistic - - import static java.nio.file.Files.createTempFile; - import static java.nio.file.Files.write; - import java.nio.file.Path; - import static java.nio.file.Paths.get; - import java.sql.Connection; - import java.sql.PreparedStatement; - import java.sql.ResultSet; - import org.postgresql.pljava.packaging.Node; - import static org.postgresql.pljava.packaging.Node.q; - import static org.postgresql.pljava.packaging.Node.stateMachine; - import static org.postgresql.pljava.packaging.Node.isVoidResultSet; - import static - org.postgresql.pljava.packaging.Node.NOTHING_OR_PGJDBC_ZERO_COUNT; - /* - * Imports that will be needed to serve a jar file over http - * when the time comes for testing that. - */ - import static java.nio.charset.StandardCharsets.UTF_8; - import java.util.jar.Attributes; - import java.util.jar.Manifest; - import java.util.jar.JarOutputStream; - import java.util.zip.ZipEntry; - import com.sun.net.httpserver.BasicAuthenticator; - import com.sun.net.httpserver.HttpContext; - import com.sun.net.httpserver.HttpExchange; - import com.sun.net.httpserver.HttpHandler; - import com.sun.net.httpserver.HttpServer; - - System.setErr(System.out); // PowerShell makes a mess of stderr output - - Node.main(new String[0]); // Extract the files (with output to stdout) - - String vmopts = "-enableassertions:org.postgresql.pljava... -Xcheck:jni"; - - if ( 17 < Runtime.version().feature() ) - vmopts += " -Djava.security.manager=allow"; - - Node n1 = Node.get_new_node("TestNode1"); - - n1.use_pg_ctl(true); - - /* - * Keep a tally of the three types of diagnostic notices that may be - * received, and, independently, how many represent no-good test results - * (error always, but also warning if seen from the tests in the - * examples.jar deployment descriptor). - */ - Map results = - Stream.of("info", "warning", "error", "ng").collect( - LinkedHashMap::new, - (m,k) -> m.put(k, 0), (r,s) -> {}); - - boolean isDiagnostic(Object o, Set whatIsNG) - { - if ( ! ( o instanceof Throwable ) ) - return false; - String[] parts = Node.classify((Throwable)o); - String type = parts[0]; - String message = parts[2]; - results.compute(type, (k,v) -> 1 + v); - if ( whatIsNG.contains(type) ) - if ( ! "warning".equals(type) || ! message.startsWith("[JEP 411]") ) - results.compute("ng", (k,v) -> 1 + v); - return true; - } - - /* - * Write a trial policy into a temporary file in n's data_dir, - * and set pljava.vmoptions accordingly over connection c. - * Returns the 'succeeding' flag from the state machine looking - * at the command results. - */ - boolean useTrialPolicy(Node n, Connection c, List contents) - throws Exception - { - Path trialPolicy = - createTempFile(n.data_dir().getParent(), "trial", "policy"); - - write(trialPolicy, contents); - - PreparedStatement setVmOpts = c.prepareStatement( - "SELECT null::pg_catalog.void" + - " FROM pg_catalog.set_config('pljava.vmoptions', ?, false)" - ); - - setVmOpts.setString(1, vmopts + - " -Dorg.postgresql.pljava.policy.trial=" + trialPolicy.toUri()); - - return stateMachine( - "change pljava.vmoptions", - null, - - q(setVmOpts, setVmOpts::execute) - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, - (o,p,q) -> null == o - ); - } - - try ( - AutoCloseable t1 = n1.initialized_cluster( - p->p.redirectErrorStream(true)); - AutoCloseable t2 = n1.started_server(Map.of( - "client_min_messages", "info", - "pljava.vmoptions", vmopts - ), p->p.redirectErrorStream(true)); - ) - { - try ( Connection c = n1.connect() ) - { - succeeding = true; // become optimistic, will be using &= below - - succeeding &= stateMachine( - "create extension no result", - null, - - q(c, "CREATE EXTENSION pljava") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - // state 1: consume any diagnostics, or show same item to state 2 - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - - NOTHING_OR_PGJDBC_ZERO_COUNT, // state 2 - - // state 3: must be end of input - (o,p,q) -> null == o - ); - } - - /* - * Get a new connection; 'create extension' always sets a near-silent - * logging level, and PL/Java only checks once at VM start time, so in - * the same session where 'create extension' was done, logging is - * somewhat suppressed. - */ - try ( Connection c = n1.connect() ) - { - succeeding &= stateMachine( - "saxon path examples path", - null, - - Node.installSaxonAndExamplesAndPath(c, - System.getProperty("mavenRepo"), - System.getProperty("saxonVer"), - true) - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - // states 1,2: diagnostics* then a void result set (saxon install) - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, - - // states 3,4: diagnostics* then a void result set (set classpath) - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 3 : -4, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 5 : false, - - // states 5,6: diagnostics* then void result set (example install) - (o,p,q) -> isDiagnostic(o, Set.of("error", "warning")) ? 5 : -6, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 7 : false, - - // states 7,8: diagnostics* then a void result set (set classpath) - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 7 : -8, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 9 : false, - - // state 9: must be end of input - (o,p,q) -> null == o - ); - - /* - * Exercise TrialPolicy some. Need another connection to change - * vmoptions. Uses some example functions, so insert here before the - * test of undeploying the examples. - */ - try ( Connection c2 = n1.connect() ) - { - succeeding &= useTrialPolicy(n1, c2, List.of( - "grant {", - " permission", - " org.postgresql.pljava.policy.TrialPolicy$Permission;", - "};" - )); - - PreparedStatement tryForbiddenRead = c2.prepareStatement( - "SELECT" + - " CASE WHEN javatest.java_getsystemproperty('java.home')" + - " OPERATOR(pg_catalog.=) ?" + - " THEN javatest.logmessage('INFO', 'trial policy test ok')" + - " ELSE javatest.logmessage('WARNING', 'trial policy test ng')" + - " END" - ); - - tryForbiddenRead.setString(1, System.getProperty("java.home")); - - succeeding &= stateMachine( - "try to read a forbidden property", - null, - - q(tryForbiddenRead, tryForbiddenRead::execute) - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error", "warning")) ? 1 : -2, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, - (o,p,q) -> null == o - ); - // done with connection c2 - } - - /* - * Spin up an http server with a little jar file to serve, and test - * that install_jar works with an http: url. - * - * First make a little jar empty but for a deployment descriptor. - */ - String ddrName = "foo.ddr"; - Attributes a = new Attributes(); - a.putValue("SQLJDeploymentDescriptor", "TRUE"); - Manifest m = new Manifest(); - m.getEntries().put(ddrName, a); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - JarOutputStream jos = new JarOutputStream(baos, m); - jos.putNextEntry(new ZipEntry(ddrName)); - jos.write( - ( - "SQLActions[]={\n\"BEGIN INSTALL\n" + - "SELECT javatest.logmessage('INFO'," + - " 'jar installed from http');\n" + - "END INSTALL\",\n\"BEGIN REMOVE\n" + - "BEGIN dummy\n" + - "END dummy;\n" + - "END REMOVE\"\n}\n" - ).getBytes(UTF_8) - ); - jos.closeEntry(); - jos.close(); - byte[] jar = baos.toByteArray(); - - /* - * Now an http server. - */ - HttpServer hs = - HttpServer.create(new InetSocketAddress("localhost", 0), 0); - - try ( - Connection c2 = n1.connect(); - AutoCloseable t = ((Supplier)() -> - { - hs.start(); - return () -> hs.stop(0); - } - ).get() - ) - { - InetSocketAddress addr = hs.getAddress(); - - String id = "bar", pw = "baz"; - - URL u = new URI( - "http", id+':'+pw, addr.getHostString(), addr.getPort(), - "/foo.jar", null, null - ).toURL(); - - HttpContext hc = hs.createContext( - u.getPath(), - new HttpHandler() - { - @Override - public void handle(HttpExchange t) throws IOException - { - try ( InputStream is = t.getRequestBody() ) { - is.readAllBytes(); - } - t.getResponseHeaders().add( - "Content-Type", "application/java-archive"); - t.sendResponseHeaders(200, jar.length); - try ( OutputStream os = t.getResponseBody() ) { - os.write(jar); - } - } - } - ); - - hc.setAuthenticator( - new BasicAuthenticator("CI realm") - // ("CI realm", UTF_8) only available in Java 14 or later - { - @Override - public boolean checkCredentials(String c_id, String c_pw) - { - return id.equals(c_id) && pw.equals(c_pw); - } - } - ); - - succeeding &= useTrialPolicy(n1, c2, List.of( - "grant codebase \"${org.postgresql.pljava.codesource}\" {", - " permission", - " java.net.URLPermission \"http:*\", \"GET:Accept\";", - "};" - )); - - succeeding &= stateMachine( - "install a jar over http", - null, - - Node.installJar(c2, u.toString(), "foo", true) - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error", "warning")) ? 1 : -2, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, - (o,p,q) -> null == o - ); - - // done with connection c2 again, and the http server - } - - /* - * Also confirm that the generated undeploy actions work. - */ - succeeding &= stateMachine( - "remove jar void result", - null, - - q(c, "SELECT sqlj.remove_jar('examples', true)") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, - (o,p,q) -> null == o - ); - - /* - * Get another new connection and make sure the extension can be - * loaded in a non-superuser session. - */ - try ( Connection c2 = n1.connect() ) - { - succeeding &= stateMachine( - "become non-superuser", - null, - - q(c2, - "CREATE ROLE alice;" + - "GRANT USAGE ON SCHEMA sqlj TO alice;" + - "SET SESSION AUTHORIZATION alice") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - NOTHING_OR_PGJDBC_ZERO_COUNT, - NOTHING_OR_PGJDBC_ZERO_COUNT, - NOTHING_OR_PGJDBC_ZERO_COUNT, - (o,p,q) -> null == o - ); - - succeeding &= stateMachine( - "load as non-superuser", - null, - - q(c2, "SELECT null::pg_catalog.void" + - " FROM sqlj.get_classpath('public')") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - (o,p,q) -> isVoidResultSet(o, 1, 1) ? 3 : false, - (o,p,q) -> null == o - ); - // done with connection c2 again - } - - /* - * Make sure the extension drops cleanly and nothing - * is left in sqlj. - */ - succeeding &= stateMachine( - "drop extension and schema no result", - null, - - q(c, "DROP EXTENSION pljava;DROP SCHEMA sqlj") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - NOTHING_OR_PGJDBC_ZERO_COUNT, - NOTHING_OR_PGJDBC_ZERO_COUNT, - (o,p,q) -> null == o - ); - } - - /* - * Get another new connection and confirm that the old, pre-extension, - * LOAD method of installing PL/Java works. It is largely obsolete in - * the era of extensions, but still covers the use case of installing - * PL/Java without admin access on the server filesystem to where - * CREATE EXTENSION requires the files to be; they can still be - * installed in some other writable location the server can read, and - * pljava.module_path set to the right locations of the jars, and the - * correct shared-object path given to LOAD. - * - * Also test the after-the-fact packaging up with CREATE EXTENSION - * FROM unpackaged. That officially goes away in PG 13, where the - * equivalent sequence - * CREATE EXTENSION pljava VERSION unpackaged - * \c - * ALTER EXTENSION pljava UPDATE - * should be tested instead. - */ - try ( Connection c = n1.connect() ) - { - int majorVersion = c.getMetaData().getDatabaseMajorVersion(); - - succeeding &= stateMachine( - "load as non-extension", - null, - - Node.loadPLJava(c) - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - NOTHING_OR_PGJDBC_ZERO_COUNT, - (o,p,q) -> null == o - ); - - if ( 13 <= majorVersion ) - { - succeeding &= stateMachine( - "create unpackaged (PG >= 13)", - null, - - q(c, "CREATE EXTENSION pljava VERSION unpackaged") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - NOTHING_OR_PGJDBC_ZERO_COUNT, - (o,p,q) -> null == o - ); - } - } - - /* - * CREATE EXTENSION FROM unpackaged (or the second half of the - * PG >= 13 CREATE EXTENSION VERSION unpackaged;ALTER EXTENSION UPDATE - * sequence) has to happen over a new connection. - */ - try ( Connection c = n1.connect() ) - { - int majorVersion = c.getMetaData().getDatabaseMajorVersion(); - - succeeding &= stateMachine( - "package after loading", - null, - - q(c, 13 > majorVersion - ? "CREATE EXTENSION pljava FROM unpackaged" - : "ALTER EXTENSION pljava UPDATE") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - NOTHING_OR_PGJDBC_ZERO_COUNT, - (o,p,q) -> null == o - ); - - /* - * Again make sure extension drops cleanly with nothing left behind. - */ - succeeding &= stateMachine( - "drop extension and schema no result", - null, - - q(c, "DROP EXTENSION pljava;DROP SCHEMA sqlj") - .flatMap(Node::semiFlattenDiagnostics) - .peek(Node::peek), - - (o,p,q) -> isDiagnostic(o, Set.of("error")) ? 1 : -2, - NOTHING_OR_PGJDBC_ZERO_COUNT, - NOTHING_OR_PGJDBC_ZERO_COUNT, - (o,p,q) -> null == o - ); - } - } catch ( Throwable t ) - { - succeeding = false; - throw t; - } - - System.out.println(results); - succeeding &= (0 == results.get("ng")); - System.exit(succeeding ? 0 : 1); - '@ | jshell ` -execution local ` "-J--class-path=$packageJar;$jdbcJar" ` @@ -579,4 +77,7 @@ test_script: "-J--add-modules=java.sql.rowset,jdk.httpserver" ` "-J-Dpgconfig=$pgConfig" ` "-J-DmavenRepo=$mavenRepo" ` - "-J-DsaxonVer=$saxonVer" - + "-J-DsaxonVer=$saxonVer" ` + "-J-DredirectError=true" ` + "-J-DextractFiles=true" ` + CI\integration From 5ca0ad36be2f79956d7fcb6c732fdc74c71a325a Mon Sep 17 00:00:00 2001 From: Chapman Flack Date: Tue, 18 Feb 2025 20:55:47 -0500 Subject: [PATCH 3/5] Use new JVM options from JEPs 471 and 472 Since JEP 471 in Java 23, --sun-misc-unsafe-memory-access=deny can be used. Since JEP 472 in Java 24, --illegal-native-access=deny can be used. Both would be best practice, especially in Java 24+ where no policy is available to control access to Unsafe or loadLibrary. Refactor the test script slightly so the Java major version and PostgreSQL major version aren't repeatedly obtained. --- CI/integration | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/CI/integration b/CI/integration index 6f3caa65a..9b0c1e066 100644 --- a/CI/integration +++ b/CI/integration @@ -126,11 +126,20 @@ Path libjvm = ( : javaLibDir.resolve(s_isWindows ? "jvm.dll" : "server/libjvm.so") ); +// Use deprecated major() here because feature() first appears in Java 10 +int jFeatureVersion = Runtime.version().major(); + String vmopts = "-enableassertions:org.postgresql.pljava... -Xcheck:jni"; -if ( 17 < Runtime.version().feature() ) +if ( 17 < jFeatureVersion ) vmopts += " -Djava.security.manager=allow"; +if ( 23 <= jFeatureVersion ) + vmopts += " --sun-misc-unsafe-memory-access=deny"; // JEP 471 + +if ( 24 <= jFeatureVersion ) + vmopts += " --illegal-native-access=deny"; // JEP 472 + Node n1 = Node.get_new_node("TestNode1"); if ( s_isWindows ) @@ -206,8 +215,12 @@ try ( ), tweaks); ) { + int pgMajorVersion; + try ( Connection c = n1.connect() ) { + pgMajorVersion = c.getMetaData().getDatabaseMajorVersion(); + succeeding = true; // become optimistic, will be using &= below succeeding &= stateMachine( @@ -511,8 +524,6 @@ try ( */ try ( Connection c = n1.connect() ) { - int majorVersion = c.getMetaData().getDatabaseMajorVersion(); - succeeding &= stateMachine( "load as non-extension", null, @@ -526,7 +537,7 @@ try ( (o,p,q) -> null == o ); - if ( 13 <= majorVersion ) + if ( 13 <= pgMajorVersion ) { succeeding &= stateMachine( "create unpackaged (PG >= 13)", @@ -550,13 +561,11 @@ try ( */ try ( Connection c = n1.connect() ) { - int majorVersion = c.getMetaData().getDatabaseMajorVersion(); - succeeding &= stateMachine( "package after loading", null, - q(c, 13 > majorVersion + q(c, 13 > pgMajorVersion ? "CREATE EXTENSION pljava FROM unpackaged" : "ALTER EXTENSION pljava UPDATE") .flatMap(Node::semiFlattenDiagnostics) From 6c8f272f2ea9ccbd77bb981fc65008cb20a47576 Mon Sep 17 00:00:00 2001 From: Chapman Flack Date: Wed, 19 Feb 2025 19:25:17 -0500 Subject: [PATCH 4/5] More, necessary, GitHub Actions workflow polishing None of this is related at all to JEP 472, but some is needed to catch up with changes on the GitHub runners as the CI would otherwise fail. The macos-12 runner is gone, and it was the last macos one that supplied PostgreSQL preinstalled. So rename the workflow from ci-runnerpg to ci-lazypg ... it still uses the preinstalled PG on runners that have one, but now resorts to brew to install it on macos. The macos-13 runner is still Intel-based (and needs PG installed). The macos-14 runner is ARM64-based (and needs PG installed). The runners have some Java versions preinstalled, so check for those and only fetch a JDK if the matrix.java version isn't one of the preinstalled ones. Preinstalled versions can be found by their JAVA_HOME_$VERSION_$ARCH environment variables, except that when $ARCH is ARM64 the environment variable has arm in lowercase. So much for consistency. There are newer versions of the setup-java and checkout actions. The old ones elicited warnings about a deprecated Node.js version. The old setup-java, when running on macos ARM64 (or arm64), would happily install an x86_64 JDK. Which would seem to work fine, apparently running in a compatibility mode, apparently inherited by the C compiler and linker spawned from Maven. The linker would then report baffling errors about not finding symbols in PostgreSQL for x86_64! This is the exact sort of mystery seen and not yet solved in issues #411 and #428. So, running Maven on a JDK that's for the wrong architecture is now a known way to cause such a problem, and worth checking in such a case. Added some ::group:: / ::endgroup:: to make the log output more easily perused. --- .../{ci-runnerpg.yml => ci-lazypg.yml} | 167 ++++++++++++++---- 1 file changed, 128 insertions(+), 39 deletions(-) rename .github/workflows/{ci-runnerpg.yml => ci-lazypg.yml} (56%) diff --git a/.github/workflows/ci-runnerpg.yml b/.github/workflows/ci-lazypg.yml similarity index 56% rename from .github/workflows/ci-runnerpg.yml rename to .github/workflows/ci-lazypg.yml index 9634ba52c..9db85e48c 100644 --- a/.github/workflows/ci-runnerpg.yml +++ b/.github/workflows/ci-lazypg.yml @@ -1,9 +1,10 @@ -# This workflow will build and test PL/Java against the version of PostgreSQL -# preinstalled in the GitHub Actions runner environment. Naturally, this one -# does not have a PostgreSQL version in the build matrix. The version that's -# preinstalled is the version you get. +# This workflow will build and test PL/Java against a version of PostgreSQL +# lazily obtained (either preinstalled in the GitHub Actions runner environment, +# or obtained from a package repository if the runner does not provide one). +# Arrange for the matrix to include a pg version, for cases where one must be +# installed. -name: CI with runner-supplied PostgreSQL version +name: CI lazy getting PostgreSQL permissions: contents: read @@ -25,43 +26,106 @@ jobs: oscc: - os: ubuntu-latest cc: gcc - - os: macos-12 + - os: macos-13 cc: clang + pg: 17 + - os: macos-14 + cc: clang + pg: 17 # - os: windows-latest # cc: msvc # - os: windows-latest # cc: mingw - java: [9, 11, 17, 19, 21] + java: [9, 11, 17, 21, 23] steps: - - name: Check out PL/Java - uses: actions/checkout@v2 - with: - path: pljava + - name: Check for JDK preinstalled + id: jdkcheck + shell: bash + env: + JAVAVER: ${{ matrix.java }} + run: | + if + candidate="JAVA_HOME_${JAVAVER}_${RUNNER_ARCH}" + echo -n "Environment contains $candidate? " + [[ -n ${!candidate+set} ]] + then + echo yes + echo >>"$GITHUB_ENV" "JAVA_HOME=${!candidate}" + echo >>"$GITHUB_OUTPUT" java_found=true + elif + candidate="JAVA_HOME_${JAVAVER}_${RUNNER_ARCH,,[A-Z]}" + echo -ne 'no\n'"Environment contains $candidate? " + [[ -n ${!candidate+set} ]] + then + echo yes + echo >>"$GITHUB_ENV" "JAVA_HOME=${!candidate}" + echo >>"$GITHUB_OUTPUT" java_found=true + else + echo -e 'no\n'"only: ${!JAVA_HOME_*}" + echo >>"$GITHUB_OUTPUT" java_found=false + fi - - name: Set up JDK - uses: actions/setup-java@v1 + - name: Fetch a JDK + if: ${{ 'false' == steps.jdkcheck.outputs.java_found }} + uses: actions/setup-java@7a6d8a8234af8eb26422e24e3006232cccaa061b with: + distribution: temurin java-version: ${{ matrix.java }} - - name: Report Java, Maven, and PostgreSQL versions (Linux, macOS) - if: ${{ 'Windows' != runner.os }} + - name: Compute absolute paths for java and jshell + shell: bash run: | - java -version - mvn --version - pg_config + if [[ $RUNNER_OS == Windows ]] + then + echo >>"$GITHUB_ENV" "ABS_JAVA=$JAVA_HOME"'\bin\java' + echo >>"$GITHUB_ENV" "ABS_JSHELL=$JAVA_HOME"'\bin\jshell' + else + echo >>"$GITHUB_ENV" "ABS_JAVA=$JAVA_HOME/bin/java" + echo >>"$GITHUB_ENV" "ABS_JSHELL=$JAVA_HOME/bin/jshell" + fi - - name: Report Java, Maven, and PostgreSQL versions (Windows) - if: ${{ 'Windows' == runner.os }} + - name: Set PGCONFIG in environment, installing PostgreSQL if needed + shell: bash + env: + PGVER: ${{ matrix.oscc.pg }} run: | - java -version + if [[ $RUNNER_OS == Linux ]] + then + echo >>"$GITHUB_ENV" PGCONFIG=pg_config + elif [[ $RUNNER_OS == Windows ]] + then + echo >>"$GITHUB_ENV" PGCONFIG="$PGBIN"'\pg_config' + elif [[ $RUNNER_OS == macOS ]] + then + echo '::group::brew update' + brew update + echo '::endgroup::' + echo "::group::brew install postgresql@$PGVER" + # HOMEBREW_GITHUB_ACTIONS will suppress the formula's initdb + HOMEBREW_GITHUB_ACTIONS=1 HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 \ + brew install postgresql@"$PGVER" + echo '::endgroup::' + pfx=$(brew --prefix postgresql@"$PGVER") + echo >>"$GITHUB_ENV" PGCONFIG="$pfx/bin/pg_config" + fi + + - name: Report Java, Maven, and PostgreSQL versions + shell: bash + run: | + "$ABS_JAVA" -version mvn --version - & "$Env:PGBIN\pg_config" + "$PGCONFIG" - name: Obtain PG development files (Ubuntu, PGDG) if: ${{ 'Linux' == runner.os }} run: | + pgver=$("$PGCONFIG" --version) + pgver=${pgver##PostgreSQL } + pgver=${pgver%% *} + pgver=${pgver%.*} + echo '::group::Install PGDG key and repo' curl -s -S https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | sudo dd of=/etc/apt/trusted.gpg.d/apt.postgresql.org.gpg @@ -71,14 +135,47 @@ jobs: "$(lsb_release -cs)-pgdg" \ main | sudo tee /etc/apt/sources.list.d/pgdg.list + echo '::endgroup::' + echo '::group::apt-get update' sudo apt-get update - sudo apt-get install postgresql-server-dev-14 libkrb5-dev + echo '::endgroup::' + echo "::group::apt-get install postgresql-server-dev-$pgver" + sudo apt-get install postgresql-server-dev-"$pgver" libkrb5-dev + echo '::endgroup::' + + - name: Confirm PostgreSQL development files are present + shell: python + run: | + from os import getenv + from os.path import join + from re import sub + from subprocess import check_output + + pgconfig = getenv('PGCONFIG') + + def ask_pg_config(what): + return check_output([pgconfig, '--'+what]).splitlines()[0] + + pgch = join(ask_pg_config('includedir-server'), b'pg_config.h') + + with open(pgch, 'r') as f: + line = [ln for ln in f if ln.startswith('#define PG_VERSION_STR ')][0] + + vers = sub(r'#define PG_VERSION_STR "(.*)"\n', r'\1', line) + + print('PostgreSQL development files are present:', vers, sep='\n') + + - name: Check out PL/Java + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + path: pljava - name: Build PL/Java (Linux, macOS) if: ${{ 'Windows' != runner.os }} working-directory: pljava run: | mvn clean install --batch-mode \ + -Dpgsql.pgconfig="$PGCONFIG" \ -Psaxon-examples -Ppgjdbc \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn @@ -126,17 +223,14 @@ jobs: run: | HOME=$( (cd .. && pwd) ) "$M2"/mvn clean install --batch-mode \ - -Dpgsql.pgconfig="$PGBIN"'\pg_config' \ + -Dpgsql.pgconfig="$PGCONFIG" \ -Psaxon-examples -Ppgjdbc \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn - name: Install and test PL/Java - if: ${{ '9' != matrix.java || 'Windows' != runner.os }} working-directory: pljava shell: bash run: | - pgConfig=pg_config # runner-supplied, just get it from the PATH - packageJar=$(find pljava-packaging -name pljava-pg*.jar -print) mavenRepo="$HOME/.m2/repository" @@ -166,22 +260,16 @@ jobs: # used when starting PostgreSQL; pg_ctl has a Windows-specific ability # to drop admin privs so postgres will not refuse to start). # - # The Windows runner seems to have an extra pg_config somewhere on the - # path, that reports it was built with MinGW and installed in paths - # containing Strawberry that don't really exist. $PGBIN\pg_config refers - # to a different build made with MSVC, and those directories really - # exist, so specify that one explicitly when running on Windows. - # # The Git for Windows bash environment includes a find command, and the # things found have unixy paths returned. Make them Windowsy here, with # a hardcoded assumption they start with /c which should become c: (as # appears to be the case in the Windows runner currently). # + echo '::group::Install files from the package jar' if [[ $RUNNER_OS == Windows ]] then pathSep=';' - pgConfig="$PGBIN"'\pg_config' - java -Dpgconfig="$pgConfig" -jar "$packageJar" + "$ABS_JAVA" -Dpgconfig="$PGCONFIG" -jar "$packageJar" function toWindowsPath() { local p p="c:${1#/c}" @@ -191,15 +279,16 @@ jobs: mavenRepo="$(toWindowsPath "$mavenRepo")" else pathSep=':' - sudo "$JAVA_HOME"/bin/java -Dpgconfig="$pgConfig" -jar "$packageJar" + sudo "$ABS_JAVA" -Dpgconfig="$PGCONFIG" -jar "$packageJar" fi + echo '::endgroup::' - jshell \ + "$ABS_JSHELL" \ -execution local \ "-J--class-path=$packageJar$pathSep$jdbcJar" \ "--class-path=$packageJar" \ "-J--add-modules=java.sql.rowset,jdk.httpserver" \ - "-J-Dpgconfig=$pgConfig" \ + "-J-Dpgconfig=$PGCONFIG" \ "-J-DmavenRepo=$mavenRepo" \ "-J-DsaxonVer=$saxonVer" \ - CI/integration + CI/integration From f2894cd007d1d9899d1167bf6ebeab93f8d44c31 Mon Sep 17 00:00:00 2001 From: Chapman Flack Date: Wed, 19 Feb 2025 19:24:06 -0500 Subject: [PATCH 5/5] There's a version of bash that can't ${foo,, ? Also, seems there's no longer a 9 (or 10) JDK version either preinstalled or fetchable for a GitHub runner. --- .github/workflows/ci-lazypg.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-lazypg.yml b/.github/workflows/ci-lazypg.yml index 9db85e48c..c11ac9104 100644 --- a/.github/workflows/ci-lazypg.yml +++ b/.github/workflows/ci-lazypg.yml @@ -36,7 +36,7 @@ jobs: # cc: msvc # - os: windows-latest # cc: mingw - java: [9, 11, 17, 21, 23] + java: [11, 17, 21, 23] steps: @@ -55,7 +55,7 @@ jobs: echo >>"$GITHUB_ENV" "JAVA_HOME=${!candidate}" echo >>"$GITHUB_OUTPUT" java_found=true elif - candidate="JAVA_HOME_${JAVAVER}_${RUNNER_ARCH,,[A-Z]}" + candidate="JAVA_HOME_${JAVAVER}_$(tr A-Z a-z <<<${RUNNER_ARCH})" echo -ne 'no\n'"Environment contains $candidate? " [[ -n ${!candidate+set} ]] then