From aab88a8b57e5fcf04bffb14d1f9906f2825723a0 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Thu, 13 Nov 2025 17:35:03 -0800 Subject: [PATCH 1/3] Reconcile protein sequences, vectors --- .../labkey/api/exp/api/ExperimentService.java | 4 +++ .../experiment/api/ClosureQueryHelper.java | 28 ++++++++----------- .../experiment/api/ExperimentServiceImpl.java | 18 ++++++++---- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/api/src/org/labkey/api/exp/api/ExperimentService.java b/api/src/org/labkey/api/exp/api/ExperimentService.java index d317517c57b..b3d123a191a 100644 --- a/api/src/org/labkey/api/exp/api/ExperimentService.java +++ b/api/src/org/labkey/api/exp/api/ExperimentService.java @@ -1183,6 +1183,10 @@ List getExpProtocolsWithParameterValue( Map> getDomainMetrics(); + void clearAncestors(ExpRunItem runItem); + void clearDataAncestors(Collection dataRowIds); + void clearMaterialAncestors(Collection materialRowIds); + class XarExportOptions { String _lsidRelativizer = LSID_OPTION_FOLDER_RELATIVE; diff --git a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java index f2daaf48d46..0d5313b38fd 100644 --- a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java +++ b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java @@ -349,26 +349,26 @@ private static void incrementalRecomputeFromTempTable(SQLFragment familyTempTabl } } - public static void clearAncestorsForMaterial(long rowId) + public static void clearAncestorsForMaterials(Collection materialRowIds) { var tx = getScope().getCurrentTransaction(); if (null != tx) { - tx.addCommitTask(() -> clearAncestorsForMaterial(rowId), DbScope.CommitTaskOption.POSTCOMMIT); + tx.addCommitTask(() -> clearAncestorsForMaterials(materialRowIds), DbScope.CommitTaskOption.POSTCOMMIT); return; } - recomputeMaterialAncestors(rowId); + recomputeMaterialAncestors(materialRowIds); } - public static void clearAncestorsForDataObject(long rowId) + public static void clearAncestorsForDataObjects(Collection dataRowIds) { var tx = getScope().getCurrentTransaction(); if (null != tx) { - tx.addCommitTask(() -> clearAncestorsForDataObject(rowId), DbScope.CommitTaskOption.POSTCOMMIT); + tx.addCommitTask(() -> clearAncestorsForDataObjects(dataRowIds), DbScope.CommitTaskOption.POSTCOMMIT); return; } - recomputeDataObjectAncestors(rowId); + recomputeDataObjectAncestors(dataRowIds); } public static void populateMaterialAncestors(Logger logger) @@ -485,11 +485,11 @@ public static void recomputeFromSeeds(SQLFragment selectSeedsSql, boolean isSamp } } - private static void recomputeMaterialAncestors(long rowId) + private static void recomputeMaterialAncestors(Collection rowIds) { SQLFragment selectSeedsSql = new SQLFragment() .append("SELECT m.RowId, m.ObjectId, 'm' AS ObjectType FROM exp.material m\n") - .append("WHERE m.RowId = ").appendValue(rowId); + .append("WHERE m.RowId ").appendInClause(rowIds, getScope().getSqlDialect()); recomputeFromSeeds(selectSeedsSql, true); } @@ -525,18 +525,16 @@ private static void recomputeMaterialAncestorsForRuns(Collection runIds) .append("SELECT m.RowId, m.ObjectId, 'm' AS ObjectType FROM exp.material m\n") .append("INNER JOIN exp.MaterialInput mi ON m.rowId = mi.materialId\n") .append("INNER JOIN exp.ProtocolApplication pa ON mi.TargetApplicationId = pa.RowId\n") - .append("WHERE pa.RunId "); - getScope().getSqlDialect().appendInClauseSql(selectSeedsSql, runIds); - selectSeedsSql + .append("WHERE pa.RunId ").appendInClause(runIds, getScope().getSqlDialect()) .append(" AND pa.CpasType = ").appendValue(ExperimentRunOutput); recomputeFromSeeds(selectSeedsSql, true); } - private static void recomputeDataObjectAncestors(long rowId) + private static void recomputeDataObjectAncestors(Collection rowIds) { SQLFragment selectSeedsSql = new SQLFragment() .append("SELECT d.RowId, d.ObjectId, 'd' AS ObjectType FROM exp.data d\n") - .append("WHERE d.RowId = ").appendValue(rowId); + .append("WHERE d.RowId ").appendInClause(rowIds, getScope().getSqlDialect()); recomputeFromSeeds(selectSeedsSql, false); } @@ -572,9 +570,7 @@ private static void recomputeDataAncestorsForRuns(Collection runIds) .append("SELECT d.RowId, d.ObjectId, 'd' AS ObjectType FROM exp.data d\n") .append("INNER JOIN exp.DataInput di ON d.rowId = di.dataId\n") .append("INNER JOIN exp.ProtocolApplication pa ON di.TargetApplicationId = pa.RowId\n") - .append("WHERE pa.RunId "); - getScope().getSqlDialect().appendInClauseSql(selectSeedsSql, runIds); - selectSeedsSql + .append("WHERE pa.RunId ").appendInClause(runIds, getScope().getSqlDialect()) .append(" AND pa.CpasType = ").appendValue(ExperimentRunOutput); recomputeFromSeeds(selectSeedsSql, false); } diff --git a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java index e3bc1bbb24b..bb226ab24f7 100644 --- a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java @@ -236,7 +236,6 @@ import org.labkey.api.view.HttpView; import org.labkey.api.view.JspTemplate; import org.labkey.api.view.JspView; -import org.labkey.api.view.NotFoundException; import org.labkey.api.view.UnauthorizedException; import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.api.view.ViewContext; @@ -244,7 +243,6 @@ import org.labkey.experiment.FileLinkFileListener; import org.labkey.experiment.MissingFilesCheckInfo; import org.labkey.experiment.XarExportType; -import org.labkey.experiment.XarExporter; import org.labkey.experiment.XarReader; import org.labkey.experiment.api.property.DomainPropertyManager; import org.labkey.experiment.controllers.exp.ExperimentController; @@ -253,13 +251,11 @@ import org.labkey.experiment.pipeline.ExperimentPipelineJob; import org.labkey.experiment.pipeline.MoveRunsPipelineJob; import org.labkey.experiment.xar.AutoFileLSIDReplacer; -import org.labkey.experiment.xar.XarExportSelection; import org.labkey.vfs.FileLike; import org.springframework.dao.DataIntegrityViolationException; import org.springframework.dao.PessimisticLockingFailureException; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.UncheckedIOException; import java.net.MalformedURLException; @@ -3615,9 +3611,19 @@ public void clearAncestors(ExpRunItem runItem) { boolean isSample = runItem instanceof ExpMaterial; if (isSample) - ClosureQueryHelper.clearAncestorsForMaterial(runItem.getRowId()); + clearMaterialAncestors(List.of(runItem.getRowId())); else - ClosureQueryHelper.clearAncestorsForDataObject(runItem.getRowId()); + clearDataAncestors(List.of(runItem.getRowId())); + } + + public void clearDataAncestors(Collection dataRowIds) + { + ClosureQueryHelper.clearAncestorsForDataObjects(dataRowIds); + } + + public void clearMaterialAncestors(Collection materialRowIds) + { + ClosureQueryHelper.clearAncestorsForMaterials(materialRowIds); } public List getProtocolApplicationsForRun(long runId) From b37388d8f01e30533b084a193318123f7955781e Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Fri, 14 Nov 2025 15:39:59 -0800 Subject: [PATCH 2/3] Recreate exp.dataancestors table upon upgrade --- .../labkey/api/exp/api/ExperimentService.java | 1 + .../experiment/api/ClosureQueryHelper.java | 35 ++++++++++++------- .../experiment/api/ExperimentServiceImpl.java | 5 +++ .../controllers/exp/ExperimentController.java | 2 +- 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/api/src/org/labkey/api/exp/api/ExperimentService.java b/api/src/org/labkey/api/exp/api/ExperimentService.java index b3d123a191a..faab8adbba7 100644 --- a/api/src/org/labkey/api/exp/api/ExperimentService.java +++ b/api/src/org/labkey/api/exp/api/ExperimentService.java @@ -1186,6 +1186,7 @@ List getExpProtocolsWithParameterValue( void clearAncestors(ExpRunItem runItem); void clearDataAncestors(Collection dataRowIds); void clearMaterialAncestors(Collection materialRowIds); + void truncateAndRecreate(@Nullable Logger logger, boolean materialAncestors, boolean dataAncestors); class XarExportOptions { diff --git a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java index 0d5313b38fd..97a76c25b67 100644 --- a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java +++ b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java @@ -49,7 +49,7 @@ public class ClosureQueryHelper { - private final static Logger logger = LogHelper.getLogger(ClosureQueryHelper.class, "Sample and data class object ancestor data computation."); + private final static Logger LOG = LogHelper.getLogger(ClosureQueryHelper.class, "Sample and data class object ancestor data computation."); final static String CONCEPT_URI = "http://www.labkey.org/types#ancestorLookup"; // N.B., This should be twice the number of generations we expect as a maximum number of ancestors due to the run nodes. @@ -309,7 +309,7 @@ private static void incrementalRecomputeFromTempTable(SQLFragment familyTempTabl SQLFragment selectInto = selectIntoTempTableSql(getScope().getSqlDialect(), from, tableNameSql); selectInto.addTempToken(ref); int count = new SqlExecutor(getScope()).execute(selectInto); - logger.debug("Selected {} rows into temp.{} for recompute of {} ancestors.", count, tempTableName, isSampleType ? "sample" : "data"); + LOG.debug("Selected {} rows into temp.{} for recompute of {} ancestors.", count, tempTableName, isSampleType ? "sample" : "data"); SQLFragment upsert; TableInfo tInfo = isSampleType ? ExperimentServiceImpl.get().getTinfoMaterialAncestors() : ExperimentServiceImpl.get().getTinfoDataAncestors(); @@ -418,22 +418,31 @@ public static void populateDataAncestors(Logger logger) logger.info("Finished populating exp.dataAncestors"); } - public static void truncateAndRecreate() + public static void truncateAndRecreate(@Nullable Logger logger) { - truncateAndRecreate(logger); + truncateAndRecreate(logger, true, true); } - public static void truncateAndRecreate(Logger logger) + public static void truncateAndRecreate(@Nullable Logger logger, boolean materialAncestors, boolean dataAncestors) { try (DbScope.Transaction transaction = ExperimentService.get().getSchema().getScope().ensureTransaction()) { - TableInfo tInfo = ExperimentServiceImpl.get().getTinfoMaterialAncestors(); - new SqlExecutor(tInfo.getSchema()).execute("TRUNCATE TABLE " + tInfo); - ClosureQueryHelper.populateMaterialAncestors(logger); + if (logger == null) + logger = LOG; - tInfo = ExperimentServiceImpl.get().getTinfoDataAncestors(); - new SqlExecutor(tInfo.getSchema()).execute("TRUNCATE TABLE " + tInfo); - ClosureQueryHelper.populateDataAncestors(logger); + if (materialAncestors) + { + TableInfo tInfo = ExperimentServiceImpl.get().getTinfoMaterialAncestors(); + new SqlExecutor(tInfo.getSchema()).execute("TRUNCATE TABLE " + tInfo); + ClosureQueryHelper.populateMaterialAncestors(logger); + } + + if (dataAncestors) + { + TableInfo tInfo = ExperimentServiceImpl.get().getTinfoDataAncestors(); + new SqlExecutor(tInfo.getSchema()).execute("TRUNCATE TABLE " + tInfo); + ClosureQueryHelper.populateDataAncestors(logger); + } transaction.commit(); } @@ -456,7 +465,7 @@ public static void recomputeFromSeeds(SQLFragment selectSeedsSql, boolean isSamp // complete hack to get SQLServer to not make RowId an identity column in the target table so the subsequent insert will work without complaint selectIntoSql.append(" UNION ALL SELECT RowId, ObjectId, 'x' AS ObjectType FROM " ).append(isSampleType ? "exp.material" : "exp.data").append(" WHERE 1 <> 1"); int numSeeds = new SqlExecutor(getScope()).execute(selectIntoSql); - logger.debug("Added {} seed {} rows to temp.{}", numSeeds, isSampleType ? "sample" : "data", familyTableName); + LOG.debug("Added {} seed {} rows to temp.{}", numSeeds, isSampleType ? "sample" : "data", familyTableName); // if we didn't actually insert any items into the table, there's nothing more to be done if (numSeeds == 0) return; @@ -472,7 +481,7 @@ public static void recomputeFromSeeds(SQLFragment selectSeedsSql, boolean isSamp descendants.append("INSERT INTO temp.").append(tableNameSql) .append(" (RowId, ObjectId, ObjectType) ").append(descendantClosureSelectSql); int numRows = new SqlExecutor(getScope()).execute(descendants); - logger.debug("Added {} {} descendant rows for {}", numRows, isSampleType ? "sample" : "data", familyTableName); + LOG.debug("Added {} {} descendant rows for {}", numRows, isSampleType ? "sample" : "data", familyTableName); // recompute the ancestors for the seed ids and the descendants incrementalRecomputeFromTempTable(tableNameSql, isSampleType); diff --git a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java index bb226ab24f7..ae064ded223 100644 --- a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java @@ -3626,6 +3626,11 @@ public void clearMaterialAncestors(Collection materialRowIds) ClosureQueryHelper.clearAncestorsForMaterials(materialRowIds); } + public void truncateAndRecreate(@Nullable Logger logger, boolean materialAncestors, boolean dataAncestors) + { + ClosureQueryHelper.truncateAndRecreate(logger, materialAncestors, dataAncestors); + } + public List getProtocolApplicationsForRun(long runId) { SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("RunId"), runId); diff --git a/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java b/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java index 09ab704bce0..036ec759e16 100644 --- a/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java +++ b/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java @@ -7388,7 +7388,7 @@ public static class RebuildAncestorsAction extends MutatingApiAction @Override public Object execute(Object form, BindException errors) { - ClosureQueryHelper.truncateAndRecreate(); + ClosureQueryHelper.truncateAndRecreate(null); return success(); } } From f031df22e7c3709147bc3eeecd7f806f0c5239a5 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Fri, 14 Nov 2025 18:31:20 -0800 Subject: [PATCH 3/3] repopulateAncestors --- .../labkey/api/exp/api/ExperimentService.java | 3 +- .../experiment/api/ClosureQueryHelper.java | 36 +++++++------------ .../experiment/api/ExperimentServiceImpl.java | 10 +++--- .../controllers/exp/ExperimentController.java | 2 +- 4 files changed, 20 insertions(+), 31 deletions(-) diff --git a/api/src/org/labkey/api/exp/api/ExperimentService.java b/api/src/org/labkey/api/exp/api/ExperimentService.java index faab8adbba7..2ce76f42943 100644 --- a/api/src/org/labkey/api/exp/api/ExperimentService.java +++ b/api/src/org/labkey/api/exp/api/ExperimentService.java @@ -82,7 +82,6 @@ import org.labkey.api.util.Pair; import org.labkey.api.util.StringUtilsLabKey; import org.labkey.api.view.HttpView; -import org.labkey.api.view.NotFoundException; import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.api.view.ViewContext; import org.labkey.vfs.FileLike; @@ -1186,7 +1185,7 @@ List getExpProtocolsWithParameterValue( void clearAncestors(ExpRunItem runItem); void clearDataAncestors(Collection dataRowIds); void clearMaterialAncestors(Collection materialRowIds); - void truncateAndRecreate(@Nullable Logger logger, boolean materialAncestors, boolean dataAncestors); + void repopulateAncestors(); class XarExportOptions { diff --git a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java index 97a76c25b67..909ff74ed46 100644 --- a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java +++ b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java @@ -17,6 +17,7 @@ import org.labkey.api.data.MutableColumnInfo; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.Table; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TempTableTracker; import org.labkey.api.data.VirtualTable; @@ -49,7 +50,7 @@ public class ClosureQueryHelper { - private final static Logger LOG = LogHelper.getLogger(ClosureQueryHelper.class, "Sample and data class object ancestor data computation."); + private final static Logger logger = LogHelper.getLogger(ClosureQueryHelper.class, "Sample and data class object ancestor data computation."); final static String CONCEPT_URI = "http://www.labkey.org/types#ancestorLookup"; // N.B., This should be twice the number of generations we expect as a maximum number of ancestors due to the run nodes. @@ -309,7 +310,7 @@ private static void incrementalRecomputeFromTempTable(SQLFragment familyTempTabl SQLFragment selectInto = selectIntoTempTableSql(getScope().getSqlDialect(), from, tableNameSql); selectInto.addTempToken(ref); int count = new SqlExecutor(getScope()).execute(selectInto); - LOG.debug("Selected {} rows into temp.{} for recompute of {} ancestors.", count, tempTableName, isSampleType ? "sample" : "data"); + logger.debug("Selected {} rows into temp.{} for recompute of {} ancestors.", count, tempTableName, isSampleType ? "sample" : "data"); SQLFragment upsert; TableInfo tInfo = isSampleType ? ExperimentServiceImpl.get().getTinfoMaterialAncestors() : ExperimentServiceImpl.get().getTinfoDataAncestors(); @@ -418,31 +419,20 @@ public static void populateDataAncestors(Logger logger) logger.info("Finished populating exp.dataAncestors"); } - public static void truncateAndRecreate(@Nullable Logger logger) + public static void truncateAndRecreate() { - truncateAndRecreate(logger, true, true); + truncateAndRecreate(logger); } - public static void truncateAndRecreate(@Nullable Logger logger, boolean materialAncestors, boolean dataAncestors) + public static void truncateAndRecreate(Logger logger) { - try (DbScope.Transaction transaction = ExperimentService.get().getSchema().getScope().ensureTransaction()) + try (DbScope.Transaction transaction = ExperimentService.get().ensureTransaction()) { - if (logger == null) - logger = LOG; + Table.truncate(ExperimentServiceImpl.get().getTinfoMaterialAncestors()); + ClosureQueryHelper.populateMaterialAncestors(logger); - if (materialAncestors) - { - TableInfo tInfo = ExperimentServiceImpl.get().getTinfoMaterialAncestors(); - new SqlExecutor(tInfo.getSchema()).execute("TRUNCATE TABLE " + tInfo); - ClosureQueryHelper.populateMaterialAncestors(logger); - } - - if (dataAncestors) - { - TableInfo tInfo = ExperimentServiceImpl.get().getTinfoDataAncestors(); - new SqlExecutor(tInfo.getSchema()).execute("TRUNCATE TABLE " + tInfo); - ClosureQueryHelper.populateDataAncestors(logger); - } + Table.truncate(ExperimentServiceImpl.get().getTinfoDataAncestors()); + ClosureQueryHelper.populateDataAncestors(logger); transaction.commit(); } @@ -465,7 +455,7 @@ public static void recomputeFromSeeds(SQLFragment selectSeedsSql, boolean isSamp // complete hack to get SQLServer to not make RowId an identity column in the target table so the subsequent insert will work without complaint selectIntoSql.append(" UNION ALL SELECT RowId, ObjectId, 'x' AS ObjectType FROM " ).append(isSampleType ? "exp.material" : "exp.data").append(" WHERE 1 <> 1"); int numSeeds = new SqlExecutor(getScope()).execute(selectIntoSql); - LOG.debug("Added {} seed {} rows to temp.{}", numSeeds, isSampleType ? "sample" : "data", familyTableName); + logger.debug("Added {} seed {} rows to temp.{}", numSeeds, isSampleType ? "sample" : "data", familyTableName); // if we didn't actually insert any items into the table, there's nothing more to be done if (numSeeds == 0) return; @@ -481,7 +471,7 @@ public static void recomputeFromSeeds(SQLFragment selectSeedsSql, boolean isSamp descendants.append("INSERT INTO temp.").append(tableNameSql) .append(" (RowId, ObjectId, ObjectType) ").append(descendantClosureSelectSql); int numRows = new SqlExecutor(getScope()).execute(descendants); - LOG.debug("Added {} {} descendant rows for {}", numRows, isSampleType ? "sample" : "data", familyTableName); + logger.debug("Added {} {} descendant rows for {}", numRows, isSampleType ? "sample" : "data", familyTableName); // recompute the ancestors for the seed ids and the descendants incrementalRecomputeFromTempTable(tableNameSql, isSampleType); diff --git a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java index ae064ded223..b9b8859e1e4 100644 --- a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java @@ -3616,6 +3616,11 @@ public void clearAncestors(ExpRunItem runItem) clearDataAncestors(List.of(runItem.getRowId())); } + public void repopulateAncestors() + { + ClosureQueryHelper.truncateAndRecreate(); + } + public void clearDataAncestors(Collection dataRowIds) { ClosureQueryHelper.clearAncestorsForDataObjects(dataRowIds); @@ -3626,11 +3631,6 @@ public void clearMaterialAncestors(Collection materialRowIds) ClosureQueryHelper.clearAncestorsForMaterials(materialRowIds); } - public void truncateAndRecreate(@Nullable Logger logger, boolean materialAncestors, boolean dataAncestors) - { - ClosureQueryHelper.truncateAndRecreate(logger, materialAncestors, dataAncestors); - } - public List getProtocolApplicationsForRun(long runId) { SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("RunId"), runId); diff --git a/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java b/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java index 036ec759e16..09ab704bce0 100644 --- a/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java +++ b/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java @@ -7388,7 +7388,7 @@ public static class RebuildAncestorsAction extends MutatingApiAction @Override public Object execute(Object form, BindException errors) { - ClosureQueryHelper.truncateAndRecreate(null); + ClosureQueryHelper.truncateAndRecreate(); return success(); } }