From 34632a57903a5ae5cdf0ea3268644c9d8c5d78bd Mon Sep 17 00:00:00 2001 From: labkey-tchad Date: Wed, 28 Jan 2026 09:45:52 -0800 Subject: [PATCH 1/5] Throw non-matching exceptions from intentional deadlock --- api/src/org/labkey/api/data/DbScope.java | 28 ++++++++++++++---------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/api/src/org/labkey/api/data/DbScope.java b/api/src/org/labkey/api/data/DbScope.java index 50af17eb9c9..cd9e51b6796 100644 --- a/api/src/org/labkey/api/data/DbScope.java +++ b/api/src/org/labkey/api/data/DbScope.java @@ -50,7 +50,6 @@ import org.labkey.api.util.GUID; import org.labkey.api.util.LoggerWriter; import org.labkey.api.util.MemTracker; -import org.labkey.api.util.Pair; import org.labkey.api.util.ResultSetUtil; import org.labkey.api.util.SimpleLoggerWriter; import org.labkey.api.util.SkipMothershipLogging; @@ -3267,13 +3266,17 @@ public void testExtraCommit() } @Test - public void testLockTimeout() + public void testLockTimeout() throws Throwable { ReentrantLock lock1 = new ReentrantLock(); ReentrantLock lock2 = new ReentrantLock(); - Pair throwables = attemptToDeadlock(lock1, lock2, (x) -> ((TransactionImpl)x).setLockTimeout(5, TimeUnit.SECONDS)); + List throwables = attemptToDeadlock(lock1, lock2, (x) -> ((TransactionImpl)x).setLockTimeout(5, TimeUnit.SECONDS)); + + assertFalse("No exceptions from attempted deadlock.", throwables.isEmpty()); + throwables = throwables.stream().filter(t -> ! (t instanceof DeadlockPreventingException)).toList(); + if (!throwables.isEmpty()) + throw throwables.getFirst(); - assertTrue(throwables.first instanceof DeadlockPreventingException || throwables.second instanceof DeadlockPreventingException); assertFalse("Lock 1 is still locked", lock1.isLocked()); assertFalse("Lock 2 is still locked", lock2.isLocked()); } @@ -3387,25 +3390,28 @@ public String getKind() } @Test - public void testServerRowLock() + public void testServerRowLock() throws Throwable { final User user = TestContext.get().getUser(); Lock lockUser = new ServerPrimaryKeyLock(true, CoreSchema.getInstance().getTableInfoUsersData(), user.getUserId()); Lock lockHome = new ServerPrimaryKeyLock(true, CoreSchema.getInstance().getTableInfoContainers(), ContainerManager.getHomeContainer().getId()); - Pair throwables = attemptToDeadlock(lockUser, lockHome, (x) -> {}); + List throwables = attemptToDeadlock(lockUser, lockHome, (x) -> {}); - assertTrue("Unexpected exceptions: " + throwables.first + "\n" + throwables.second, throwables.first instanceof PessimisticLockingFailureException || throwables.second instanceof PessimisticLockingFailureException ); + assertFalse("No exceptions from attempted deadlock.", throwables.isEmpty()); + throwables = throwables.stream().filter(t -> ! (t instanceof PessimisticLockingFailureException)).toList(); + if (!throwables.isEmpty()) + throw throwables.getFirst(); } /** * @return foreground and background thread exceptions */ - private Pair attemptToDeadlock(Lock lock1, Lock lock2, @NotNull Consumer transactionModifier) + private List attemptToDeadlock(Lock lock1, Lock lock2, @NotNull Consumer transactionModifier) { final Object notifier = new Object(); - final Pair result = new Pair<>(null, null); + final List result = new ArrayList<>(); // let's try to intentionally cause a deadlock Thread bkg = new Thread(() -> { @@ -3431,7 +3437,7 @@ private Pair attemptToDeadlock(Lock lock1, Lock lock2, @No } catch (Throwable x) { - result.second = x; + result.add(x); } } }); @@ -3465,7 +3471,7 @@ private Pair attemptToDeadlock(Lock lock1, Lock lock2, @No } catch (Throwable x) { - result.first = x; + result.add(x); } finally { From b0973f97747a66eb3e8a424ebc6450e293c41ec4 Mon Sep 17 00:00:00 2001 From: labkey-tchad Date: Wed, 28 Jan 2026 14:38:51 -0800 Subject: [PATCH 2/5] DRY it up --- api/src/org/labkey/api/data/DbScope.java | 28 ++++++++++-------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/api/src/org/labkey/api/data/DbScope.java b/api/src/org/labkey/api/data/DbScope.java index cd9e51b6796..bff0373fc13 100644 --- a/api/src/org/labkey/api/data/DbScope.java +++ b/api/src/org/labkey/api/data/DbScope.java @@ -92,6 +92,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Properties; import java.util.Random; import java.util.RandomAccess; @@ -3270,12 +3271,7 @@ public void testLockTimeout() throws Throwable { ReentrantLock lock1 = new ReentrantLock(); ReentrantLock lock2 = new ReentrantLock(); - List throwables = attemptToDeadlock(lock1, lock2, (x) -> ((TransactionImpl)x).setLockTimeout(5, TimeUnit.SECONDS)); - - assertFalse("No exceptions from attempted deadlock.", throwables.isEmpty()); - throwables = throwables.stream().filter(t -> ! (t instanceof DeadlockPreventingException)).toList(); - if (!throwables.isEmpty()) - throw throwables.getFirst(); + attemptToDeadlock(lock1, lock2, (x) -> ((TransactionImpl)x).setLockTimeout(5, TimeUnit.SECONDS), DeadlockPreventingException.class); assertFalse("Lock 1 is still locked", lock1.isLocked()); assertFalse("Lock 2 is still locked", lock2.isLocked()); @@ -3397,18 +3393,10 @@ public void testServerRowLock() throws Throwable Lock lockUser = new ServerPrimaryKeyLock(true, CoreSchema.getInstance().getTableInfoUsersData(), user.getUserId()); Lock lockHome = new ServerPrimaryKeyLock(true, CoreSchema.getInstance().getTableInfoContainers(), ContainerManager.getHomeContainer().getId()); - List throwables = attemptToDeadlock(lockUser, lockHome, (x) -> {}); - - assertFalse("No exceptions from attempted deadlock.", throwables.isEmpty()); - throwables = throwables.stream().filter(t -> ! (t instanceof PessimisticLockingFailureException)).toList(); - if (!throwables.isEmpty()) - throw throwables.getFirst(); + attemptToDeadlock(lockUser, lockHome, (x) -> {}, PessimisticLockingFailureException.class); } - /** - * @return foreground and background thread exceptions - */ - private List attemptToDeadlock(Lock lock1, Lock lock2, @NotNull Consumer transactionModifier) + private void attemptToDeadlock(Lock lock1, Lock lock2, @NotNull Consumer transactionModifier, Class expectedException) throws Throwable { final Object notifier = new Object(); final List result = new ArrayList<>(); @@ -3485,7 +3473,13 @@ private List attemptToDeadlock(Lock lock1, Lock lock2, @NotNull Consu } } } - return result; + + assertFalse("No exception from attempted deadlock.", result.isEmpty()); + Optional unwantedException = result.stream().filter(t -> !expectedException.isAssignableFrom(t.getClass())).findAny(); + if (unwantedException.isPresent()) + { + throw unwantedException.get(); + } } @Test From 6e39eb9d7dd2c02a74eeffebf5e9a995e690588d Mon Sep 17 00:00:00 2001 From: labkey-tchad Date: Wed, 28 Jan 2026 14:43:43 -0800 Subject: [PATCH 3/5] Improve error message --- api/src/org/labkey/api/data/DbScope.java | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/api/src/org/labkey/api/data/DbScope.java b/api/src/org/labkey/api/data/DbScope.java index bff0373fc13..6293251d65b 100644 --- a/api/src/org/labkey/api/data/DbScope.java +++ b/api/src/org/labkey/api/data/DbScope.java @@ -92,7 +92,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Properties; import java.util.Random; import java.util.RandomAccess; @@ -3267,7 +3266,7 @@ public void testExtraCommit() } @Test - public void testLockTimeout() throws Throwable + public void testLockTimeout() { ReentrantLock lock1 = new ReentrantLock(); ReentrantLock lock2 = new ReentrantLock(); @@ -3386,7 +3385,7 @@ public String getKind() } @Test - public void testServerRowLock() throws Throwable + public void testServerRowLock() { final User user = TestContext.get().getUser(); @@ -3396,7 +3395,7 @@ public void testServerRowLock() throws Throwable attemptToDeadlock(lockUser, lockHome, (x) -> {}, PessimisticLockingFailureException.class); } - private void attemptToDeadlock(Lock lock1, Lock lock2, @NotNull Consumer transactionModifier, Class expectedException) throws Throwable + private void attemptToDeadlock(Lock lock1, Lock lock2, @NotNull Consumer transactionModifier, Class expectedException) { final Object notifier = new Object(); final List result = new ArrayList<>(); @@ -3474,12 +3473,9 @@ private void attemptToDeadlock(Lock lock1, Lock lock2, @NotNull Consumer unwantedException = result.stream().filter(t -> !expectedException.isAssignableFrom(t.getClass())).findAny(); - if (unwantedException.isPresent()) - { - throw unwantedException.get(); - } + assertFalse("No exceptions from attempted deadlock.", result.isEmpty()); + result.stream().filter(t -> !expectedException.isAssignableFrom(t.getClass())) + .findAny().ifPresent(t -> { throw new RuntimeException("Deadlock didn't generate expected " + expectedException.getSimpleName(), t); }); } @Test From e98ee8079c4888263e7e6862210e2facd91596d2 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Wed, 28 Jan 2026 17:16:52 -0800 Subject: [PATCH 4/5] Best-effort at Postgres lock and activity logging --- .../org/labkey/api/util/DebugInfoDumper.java | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/api/src/org/labkey/api/util/DebugInfoDumper.java b/api/src/org/labkey/api/util/DebugInfoDumper.java index c252ab34bb2..0aa02445f66 100644 --- a/api/src/org/labkey/api/util/DebugInfoDumper.java +++ b/api/src/org/labkey/api/util/DebugInfoDumper.java @@ -412,8 +412,22 @@ public static synchronized void dumpThreads(LoggerWriter logWriter) // Schema won't exist on SQLServer if (schema != null) { - writeTable(logWriter, schema, BasePostgreSqlDialect.POSTGRES_STAT_ACTIVITY_TABLE_NAME, "Postgres activity"); - writeTable(logWriter, schema, BasePostgreSqlDialect.POSTGRES_LOCKS_TABLE_NAME, "Postgres locks"); + try + { + writeTable(logWriter, schema, BasePostgreSqlDialect.POSTGRES_LOCKS_TABLE_NAME, "Postgres locks"); + } + catch (RuntimeException e) + { + logWriter.debug("Failed to write Postgres locks table:" + e); + } + try + { + writeTable(logWriter, schema, BasePostgreSqlDialect.POSTGRES_STAT_ACTIVITY_TABLE_NAME, "Postgres activity"); + } + catch (RuntimeException e) + { + logWriter.debug("Failed to write Postgres activity table:" + e); + } } } @@ -434,7 +448,7 @@ private static void writeTable(LoggerWriter logWriter, UserSchema schema, String PrintWriter printWriter = new PrintWriter(stringWriter); writer.write(printWriter); printWriter.flush(); - logWriter.debug("\n" + stringWriter.toString()); + logWriter.debug("\n" + stringWriter); } catch (IOException e) { From 1c610284466465ce4d575cb8500cfb0aa6e22173 Mon Sep 17 00:00:00 2001 From: labkey-tchad Date: Thu, 29 Jan 2026 11:07:59 -0800 Subject: [PATCH 5/5] Improve error message --- api/src/org/labkey/api/data/DbScope.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/api/src/org/labkey/api/data/DbScope.java b/api/src/org/labkey/api/data/DbScope.java index 6293251d65b..6e9663bf496 100644 --- a/api/src/org/labkey/api/data/DbScope.java +++ b/api/src/org/labkey/api/data/DbScope.java @@ -3475,7 +3475,10 @@ private void attemptToDeadlock(Lock lock1, Lock lock2, @NotNull Consumer !expectedException.isAssignableFrom(t.getClass())) - .findAny().ifPresent(t -> { throw new RuntimeException("Deadlock didn't generate expected " + expectedException.getSimpleName(), t); }); + .findAny().ifPresent(t -> { + throw new AssertionError("Wrong error from deadlock. expected: <" + + expectedException.getSimpleName() + ">, but was <" + t.getClass().getSimpleName() + ">", t); + }); } @Test