From 098e2930bb67f2923cc125364ebd5c2868ac9b65 Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Fri, 7 Nov 2025 13:10:28 -0800 Subject: [PATCH 01/12] Rework attachment handling during database migrations --- .../announcements/AnnouncementModule.java | 43 ++++++++++++++ .../announcements/model/AnnouncementType.java | 5 +- .../api/attachments/AttachmentType.java | 24 +++++++- .../api/data/DatabaseMigrationService.java | 56 ++++++++++++++++++- .../labkey/api/reports/report/ReportType.java | 4 +- .../api/security/AuthenticationLogoType.java | 5 +- .../org/labkey/api/security/AvatarType.java | 7 ++- .../core/CoreMigrationSchemaHandler.java | 40 +++++++++++-- .../DataClassMigrationSchemaHandler.java | 4 +- .../SampleTypeMigrationSchemaHandler.java | 2 +- .../issue/IssueMigrationSchemaHandler.java | 11 ++++ .../labkey/issue/model/IssueCommentType.java | 4 +- wiki/src/org/labkey/wiki/WikiModule.java | 31 ---------- wiki/src/org/labkey/wiki/model/WikiType.java | 5 +- 14 files changed, 187 insertions(+), 54 deletions(-) diff --git a/announcements/src/org/labkey/announcements/AnnouncementModule.java b/announcements/src/org/labkey/announcements/AnnouncementModule.java index 4967d394f2c..0479fa98961 100644 --- a/announcements/src/org/labkey/announcements/AnnouncementModule.java +++ b/announcements/src/org/labkey/announcements/AnnouncementModule.java @@ -31,11 +31,17 @@ import org.labkey.api.announcements.CommSchema; import org.labkey.api.announcements.api.AnnouncementService; import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.attachments.AttachmentType; import org.labkey.api.audit.AuditLogService; import org.labkey.api.audit.provider.MessageAuditProvider; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DatabaseMigrationConfiguration; +import org.labkey.api.data.DatabaseMigrationService; +import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationSchemaHandler; +import org.labkey.api.data.DbSchema; import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.TableInfo; import org.labkey.api.message.digest.DailyMessageDigest; import org.labkey.api.message.settings.MessageConfigService; import org.labkey.api.module.DefaultModule; @@ -165,6 +171,43 @@ public void doStartup(ModuleContext moduleContext) { fsr.addFactories(new NotificationSettingsWriterFactory(), new NotificationSettingsImporterFactory()); } + + // AnnouncementModule owns the schema, so it registers the schema handler... even though it's mostly about wiki + DatabaseMigrationService.get().registerSchemaHandler(new DefaultMigrationSchemaHandler(CommSchema.getInstance().getSchema()) + { + @Override + public void beforeSchema() + { + new SqlExecutor(getSchema()).execute("ALTER TABLE comm.Pages DROP CONSTRAINT FK_Pages_PageVersions"); + new SqlExecutor(getSchema()).execute("ALTER TABLE comm.Pages DROP CONSTRAINT FK_Pages_Parent"); + } + + @Override + public List getTablesToCopy() + { + List tablesToCopy = super.getTablesToCopy(); + tablesToCopy.add(CommSchema.getInstance().getTableInfoPages()); + tablesToCopy.add(CommSchema.getInstance().getTableInfoPageVersions()); + + return tablesToCopy; + } + + @Override + public void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema) + { + new SqlExecutor(getSchema()).execute("ALTER TABLE comm.Pages ADD CONSTRAINT FK_Pages_PageVersions FOREIGN KEY (PageVersionId) REFERENCES comm.PageVersions (RowId)"); + new SqlExecutor(getSchema()).execute("ALTER TABLE comm.Pages ADD CONSTRAINT FK_Pages_Parent FOREIGN KEY (Parent) REFERENCES comm.Pages (RowId)"); + } + + @Override + public @NotNull Collection getAttachmentTypes() + { + // TODO: Need a way to get WikiType in here + return List.of( + AnnouncementType.get() + ); + } + }); } diff --git a/announcements/src/org/labkey/announcements/model/AnnouncementType.java b/announcements/src/org/labkey/announcements/model/AnnouncementType.java index 5590b98fdfe..85d44b1f495 100644 --- a/announcements/src/org/labkey/announcements/model/AnnouncementType.java +++ b/announcements/src/org/labkey/announcements/model/AnnouncementType.java @@ -16,6 +16,7 @@ package org.labkey.announcements.model; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.labkey.api.announcements.CommSchema; import org.labkey.api.attachments.AttachmentType; import org.labkey.api.data.SQLFragment; @@ -40,8 +41,8 @@ public static AttachmentType get() } @Override - public void addWhereSql(SQLFragment sql, String parentColumn, String documentNameColumn) + public @Nullable SQLFragment getSelectParentEntityIdsSql() { - sql.append(parentColumn).append(" IN (SELECT EntityId FROM ").append(CommSchema.getInstance().getTableInfoAnnouncements(), "ann").append(")"); + return new SQLFragment("SELECT EntityId FROM ").append(CommSchema.getInstance().getTableInfoAnnouncements(), "ann"); } } diff --git a/api/src/org/labkey/api/attachments/AttachmentType.java b/api/src/org/labkey/api/attachments/AttachmentType.java index 51175668c0d..31510dc4e20 100644 --- a/api/src/org/labkey/api/attachments/AttachmentType.java +++ b/api/src/org/labkey/api/attachments/AttachmentType.java @@ -16,6 +16,7 @@ package org.labkey.api.attachments; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.labkey.api.data.SQLFragment; /** @@ -43,10 +44,29 @@ public void addWhereSql(SQLFragment sql, String parentColumn, String documentNam @NotNull String getUniqueName(); /** - * Append to the where clause of a query that wants to select attachments of the implementing type + * Append to the where clause of a query that wants to select attachments of the implementing type from the + * core.Documents table * @param sql Implementers MUST append a valid where clause to this SQLFragment * @param parentColumn Column identifier for use in where clause. Usually represents 'core.Documents.Parent' * @param documentNameColumn Column identifier for use in where clause. Usually represents 'core.Documents.DocumentName' */ - void addWhereSql(SQLFragment sql, String parentColumn, String documentNameColumn); + default void addWhereSql(SQLFragment sql, String parentColumn, String documentNameColumn) + { + SQLFragment selectSql = getSelectParentEntityIdsSql(); + if (selectSql == null) + throw new IllegalStateException("Must override either addWhereSql() or getSelectParentEntityIdsSql()"); + sql.append(parentColumn).append(" IN (").append(selectSql).append(")"); + } + + /** + * Return a SQLFragment that selects all the EntityIds that might be attachment parents from the table(s) that + * provide attachments of this type, without involving the core.Documents table. For example, + * {@code SELECT EntityId FROM comm.Announcements}. Return null if this is not-yet-implemented or inappropriate. + * For example, some attachments' parents are container IDs. If the method determines that no parents exist, then + * return a valid query that selects no rows, for example, {@code SELECT EntityID WHERE 1 = 0}. + */ + default @Nullable SQLFragment getSelectParentEntityIdsSql() + { + return null; + } } diff --git a/api/src/org/labkey/api/data/DatabaseMigrationService.java b/api/src/org/labkey/api/data/DatabaseMigrationService.java index b32dd40f0c9..466e4f3e30c 100644 --- a/api/src/org/labkey/api/data/DatabaseMigrationService.java +++ b/api/src/org/labkey/api/data/DatabaseMigrationService.java @@ -3,6 +3,7 @@ import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.labkey.api.attachments.AttachmentType; import org.labkey.api.data.DatabaseMigrationConfiguration.DefaultDatabaseMigrationConfiguration; import org.labkey.api.data.SimpleFilter.AndClause; import org.labkey.api.data.SimpleFilter.FilterClause; @@ -66,7 +67,8 @@ default void registerMigrationFilter(MigrationFilter filter) {} return null; } - default void copySourceTableToTargetTable(DatabaseMigrationConfiguration configuration, TableInfo sourceTable, TableInfo targetTable, DbSchemaType schemaType, MigrationSchemaHandler schemaHandler) {}; + default void copySourceTableToTargetTable(DatabaseMigrationConfiguration configuration, TableInfo sourceTable, TableInfo targetTable, DbSchemaType schemaType, boolean updateSequences, String additionalLogMessage, MigrationSchemaHandler schemaHandler) {} + default void updateSequences(TableInfo sourceTable, TableInfo targetTable) {} interface MigrationSchemaHandler { @@ -103,6 +105,11 @@ interface MigrationSchemaHandler void afterTable(TableInfo sourceTable, TableInfo targetTable, SimpleFilter notCopiedFilter); void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema); + + // TODO: Return Collection, indicating which attachment types it handled? + void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema); + + @NotNull Collection getAttachmentTypes(); } class DefaultMigrationSchemaHandler implements MigrationSchemaHandler @@ -299,6 +306,53 @@ public void afterTable(TableInfo sourceTable, TableInfo targetTable, SimpleFilte { } + @Override + public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema) + { + // Now that the table tables in this schema have been populated, copy all associated attachments. By + // default, use this handler's attachment types to select from the target tables all EntityIds that might be + // attachment parents (this avoids re-running potentially expensive queries on the source tables). Use the + // set of EntityIds to copy those attachments from the core.Documents table in the source database. Override + // if special behavior is required, for example, AttachmentTypes that use documentNameColumn since that + // requires querying and re-filtering the source tables instead. + getAttachmentTypes().forEach(type -> { + SQLFragment sql = type.getSelectParentEntityIdsSql(); + if (sql != null) + { + Collection entityIds = new SqlSelector(targetSchema, sql).getCollection(String.class); + FilterClause filterClause = new InClause(FieldKey.fromParts("Parent"), entityIds); + copyAttachments(configuration, sourceSchema, filterClause, " associated with " + type.getClass().getSimpleName()); + } + + // TODO: Mark this attachment type as having been seen + // TODO: afterMigration() and update core.Documents' sequence + // TODO: throw if some registered AttachmentType is not seen + // TODO: get rid of TableHandler + // TODO: eventually, fail if type.getSelectParentEntityIdsSql() returns null + // TODO: implement a bunch more AttachmentTypes, override for Notebooks? + }); + } + + // Copy all core.Documents rows that match the provided filter clause + protected void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, FilterClause filterClause, String additionalLogMessage) + { + TableInfo sourceDocumentsTable = sourceSchema.getScope().getSchema("core", DbSchemaType.Migration).getTable("Documents"); + TableInfo targetDocumentsTable = CoreSchema.getInstance().getTableInfoDocuments(); + DatabaseMigrationService.get().copySourceTableToTargetTable(configuration, sourceDocumentsTable, targetDocumentsTable, DbSchemaType.Module, false, additionalLogMessage, new DefaultMigrationSchemaHandler(CoreSchema.getInstance().getSchema()){ + @Override + public FilterClause getTableFilterClause(TableInfo sourceTable, Set containers) + { + return filterClause; + } + }); + } + + @Override + public @NotNull Collection getAttachmentTypes() + { + return List.of(); + } + @Override public void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema) { diff --git a/api/src/org/labkey/api/reports/report/ReportType.java b/api/src/org/labkey/api/reports/report/ReportType.java index a48245a776d..2954f9e5035 100644 --- a/api/src/org/labkey/api/reports/report/ReportType.java +++ b/api/src/org/labkey/api/reports/report/ReportType.java @@ -40,8 +40,8 @@ private ReportType() } @Override - public void addWhereSql(SQLFragment sql, String parentColumn, String documentNameColumn) + public @NotNull SQLFragment getSelectParentEntityIdsSql() { - sql.append(parentColumn).append(" IN (SELECT EntityId FROM ").append(CoreSchema.getInstance().getTableInfoReport(), "reports").append(")"); + return new SQLFragment("SELECT EntityId FROM ").append(CoreSchema.getInstance().getTableInfoReport(), "reports"); } } diff --git a/api/src/org/labkey/api/security/AuthenticationLogoType.java b/api/src/org/labkey/api/security/AuthenticationLogoType.java index d469bce41d4..b9e5dcba0dc 100644 --- a/api/src/org/labkey/api/security/AuthenticationLogoType.java +++ b/api/src/org/labkey/api/security/AuthenticationLogoType.java @@ -16,6 +16,7 @@ package org.labkey.api.security; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.labkey.api.attachments.AttachmentType; import org.labkey.api.data.CoreSchema; import org.labkey.api.data.SQLFragment; @@ -40,8 +41,8 @@ private AuthenticationLogoType() } @Override - public void addWhereSql(SQLFragment sql, String parentColumn, String documentNameColumn) + public @Nullable SQLFragment getSelectParentEntityIdsSql() { - sql.append(parentColumn).append(" IN (SELECT EntityId FROM ").append(CoreSchema.getInstance().getTableInfoAuthenticationConfigurations(), "acs").append(")"); + return new SQLFragment("SELECT EntityId FROM ").append(CoreSchema.getInstance().getTableInfoAuthenticationConfigurations(), "acs"); } } diff --git a/api/src/org/labkey/api/security/AvatarType.java b/api/src/org/labkey/api/security/AvatarType.java index 04b6c46366c..3446afc1d85 100644 --- a/api/src/org/labkey/api/security/AvatarType.java +++ b/api/src/org/labkey/api/security/AvatarType.java @@ -16,12 +16,13 @@ package org.labkey.api.security; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.labkey.api.attachments.AttachmentType; import org.labkey.api.data.CoreSchema; import org.labkey.api.data.SQLFragment; /** - * Identifies avatar files (user-account associated image/icon + * Identifies avatar (user-account associated image/icon) attachments */ public class AvatarType implements AttachmentType { @@ -43,8 +44,8 @@ private AvatarType() } @Override - public void addWhereSql(SQLFragment sql, String parentColumn, String documentNameColumn) + public @Nullable SQLFragment getSelectParentEntityIdsSql() { - sql.append(parentColumn).append(" IN (SELECT EntityId FROM ").append(CoreSchema.getInstance().getTableInfoUsers(), "users").append(")"); + return new SQLFragment("SELECT EntityId FROM ").append(CoreSchema.getInstance().getTableInfoUsers(), "users"); } } diff --git a/core/src/org/labkey/core/CoreMigrationSchemaHandler.java b/core/src/org/labkey/core/CoreMigrationSchemaHandler.java index bc33e9c6b62..271ed39f5e5 100644 --- a/core/src/org/labkey/core/CoreMigrationSchemaHandler.java +++ b/core/src/org/labkey/core/CoreMigrationSchemaHandler.java @@ -1,11 +1,15 @@ package org.labkey.core; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.labkey.api.attachments.AttachmentType; +import org.labkey.api.attachments.LookAndFeelResourceType; import org.labkey.api.data.CompareType; import org.labkey.api.data.CompareType.CompareClause; import org.labkey.api.data.CoreSchema; import org.labkey.api.data.DatabaseMigrationConfiguration; import org.labkey.api.data.DatabaseMigrationService; +import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationSchemaHandler; import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.DbScope; @@ -21,13 +25,17 @@ import org.labkey.api.data.TestSchema; import org.labkey.api.module.ModuleLoader; import org.labkey.api.query.FieldKey; +import org.labkey.api.reports.report.ReportType; +import org.labkey.api.security.AuthenticationLogoType; +import org.labkey.api.security.AvatarType; import org.labkey.api.util.ConfigurationException; import org.labkey.api.util.GUID; +import java.util.Collection; import java.util.List; import java.util.Set; -class CoreMigrationSchemaHandler extends DatabaseMigrationService.DefaultMigrationSchemaHandler implements DatabaseMigrationService.MigrationFilter +class CoreMigrationSchemaHandler extends DefaultMigrationSchemaHandler implements DatabaseMigrationService.MigrationFilter { static void register() { @@ -35,7 +43,7 @@ static void register() DatabaseMigrationService.get().registerSchemaHandler(schemaHandler); DatabaseMigrationService.get().registerMigrationFilter(schemaHandler); - DatabaseMigrationService.get().registerSchemaHandler(new DatabaseMigrationService.DefaultMigrationSchemaHandler(PropertySchema.getInstance().getSchema()){ + DatabaseMigrationService.get().registerSchemaHandler(new DefaultMigrationSchemaHandler(PropertySchema.getInstance().getSchema()){ @Override public @Nullable FieldKey getContainerFieldKey(TableInfo sourceTable) { @@ -43,7 +51,7 @@ static void register() } }); - DatabaseMigrationService.get().registerSchemaHandler(new DatabaseMigrationService.DefaultMigrationSchemaHandler(TestSchema.getInstance().getSchema()){ + DatabaseMigrationService.get().registerSchemaHandler(new DefaultMigrationSchemaHandler(TestSchema.getInstance().getSchema()){ @Override public List getTablesToCopy() { @@ -53,7 +61,7 @@ public List getTablesToCopy() if (ModuleLoader.getInstance().getModule(DbScope.getLabKeyScope(), "vehicle") != null) { - DatabaseMigrationService.get().registerSchemaHandler(new DatabaseMigrationService.DefaultMigrationSchemaHandler(DbSchema.get("vehicle", DbSchemaType.Module)) + DatabaseMigrationService.get().registerSchemaHandler(new DefaultMigrationSchemaHandler(DbSchema.get("vehicle", DbSchemaType.Module)) { @Override public List getTablesToCopy() @@ -95,6 +103,7 @@ public List getTablesToCopy() tablesToCopy.remove(CoreSchema.getInstance().getTableInfoModules()); tablesToCopy.remove(CoreSchema.getInstance().getTableInfoSqlScripts()); tablesToCopy.remove(CoreSchema.getInstance().getTableInfoUpgradeSteps()); + tablesToCopy.remove(CoreSchema.getInstance().getTableInfoDocuments()); return tablesToCopy; } @@ -180,6 +189,29 @@ public void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema s new SqlExecutor(getSchema()).execute("ALTER TABLE core.ViewCategory ADD CONSTRAINT FK_ViewCategory_Parent FOREIGN KEY (Parent) REFERENCES core.ViewCategory(RowId)"); } + @Override + public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema) + { + // Default handling for most of the attachment types + super.copyAttachments(configuration, sourceSchema, targetSchema); + + // Special handling for LookAndFeelResourceType, which must select from the source database + SQLFragment sql = new SQLFragment(); + LookAndFeelResourceType.get().addWhereSql(sql, "Parent", "DocumentName"); + + copyAttachments(configuration, sourceSchema, new SQLClause(sql), ""); + } + + @Override + public @NotNull Collection getAttachmentTypes() + { + return List.of( + AuthenticationLogoType.get(), + AvatarType.get(), + ReportType.get() + ); + } + // MigrationFilter implementation below private SQLFragment _groupFilterCondition = null; diff --git a/experiment/src/org/labkey/experiment/DataClassMigrationSchemaHandler.java b/experiment/src/org/labkey/experiment/DataClassMigrationSchemaHandler.java index 18f3c5ffbc9..140d9ccfab6 100644 --- a/experiment/src/org/labkey/experiment/DataClassMigrationSchemaHandler.java +++ b/experiment/src/org/labkey/experiment/DataClassMigrationSchemaHandler.java @@ -68,7 +68,7 @@ public FilterClause getContainerClause(TableInfo sourceTable, Set containe @Override public void addDomainDataFilterClause(OrClause orClause, DataFilter filter, TableInfo sourceTable, Set selectColumnNames) { - // Data classes have a built-in Flag field + // Data classes have an implicit Flag field if (filter.column().equalsIgnoreCase("Flag")) { addObjectPropertyClause(orClause, filter, sourceTable, getCommentPropertyId(sourceTable.getSchema().getScope())); @@ -177,7 +177,7 @@ public void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema s TableInfo sourceTable = biologicsSourceSchema.getTable("SequenceIdentity"); TableInfo targetTable = biologicsTargetSchema.getTable("SequenceIdentity"); - DatabaseMigrationService.get().copySourceTableToTargetTable(configuration, sourceTable, targetTable, DbSchemaType.Module, new DefaultMigrationSchemaHandler(biologicsTargetSchema) + DatabaseMigrationService.get().copySourceTableToTargetTable(configuration, sourceTable, targetTable, DbSchemaType.Module, true, null, new DefaultMigrationSchemaHandler(biologicsTargetSchema) { @Override public FilterClause getTableFilterClause(TableInfo sourceTable, Set containers) diff --git a/experiment/src/org/labkey/experiment/SampleTypeMigrationSchemaHandler.java b/experiment/src/org/labkey/experiment/SampleTypeMigrationSchemaHandler.java index c963d1d57e0..691cf334991 100644 --- a/experiment/src/org/labkey/experiment/SampleTypeMigrationSchemaHandler.java +++ b/experiment/src/org/labkey/experiment/SampleTypeMigrationSchemaHandler.java @@ -56,7 +56,7 @@ public void addDomainDataFilterClause(OrClause orClause, DataFilter filter, Tabl { String joinColumnName = getJoinColumnName(sourceTable); - // Select all rows where the built-in flag column equals the filter value + // Select all rows where the implicit flag column equals the filter value orClause.addClause( new SQLClause(new SQLFragment() .appendIdentifier(joinColumnName) diff --git a/issues/src/org/labkey/issue/IssueMigrationSchemaHandler.java b/issues/src/org/labkey/issue/IssueMigrationSchemaHandler.java index 90e3c4e19eb..0efa2667ee9 100644 --- a/issues/src/org/labkey/issue/IssueMigrationSchemaHandler.java +++ b/issues/src/org/labkey/issue/IssueMigrationSchemaHandler.java @@ -1,6 +1,8 @@ package org.labkey.issue; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.attachments.AttachmentType; import org.labkey.api.collections.CsvSet; import org.labkey.api.data.DatabaseMigrationConfiguration; import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationSchemaHandler; @@ -18,8 +20,11 @@ import org.labkey.api.query.FieldKey; import org.labkey.api.util.StringUtilsLabKey; import org.labkey.api.util.logging.LogHelper; +import org.labkey.issue.model.IssueCommentType; +import java.util.Collection; import java.util.HashSet; +import java.util.List; import java.util.Set; public class IssueMigrationSchemaHandler extends DefaultMigrationSchemaHandler @@ -76,4 +81,10 @@ public void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema s Table.delete(IssuesSchema.getInstance().getTableInfoIssues(), deleteFilter); } } + + @Override + public @NotNull Collection getAttachmentTypes() + { + return List.of(IssueCommentType.get()); + } } diff --git a/issues/src/org/labkey/issue/model/IssueCommentType.java b/issues/src/org/labkey/issue/model/IssueCommentType.java index d2074b563f0..88eeda3106c 100644 --- a/issues/src/org/labkey/issue/model/IssueCommentType.java +++ b/issues/src/org/labkey/issue/model/IssueCommentType.java @@ -40,8 +40,8 @@ private IssueCommentType() } @Override - public void addWhereSql(SQLFragment sql, String parentColumn, String documentNameColumn) + public @NotNull SQLFragment getSelectParentEntityIdsSql() { - sql.append(parentColumn).append(" IN (SELECT EntityId FROM ").append(IssuesSchema.getInstance().getTableInfoComments(), "comments").append(")"); + return new SQLFragment("SELECT EntityId FROM ").append(IssuesSchema.getInstance().getTableInfoComments(), "comments"); } } diff --git a/wiki/src/org/labkey/wiki/WikiModule.java b/wiki/src/org/labkey/wiki/WikiModule.java index 2287744d497..6e6f4c4e18e 100644 --- a/wiki/src/org/labkey/wiki/WikiModule.java +++ b/wiki/src/org/labkey/wiki/WikiModule.java @@ -25,12 +25,7 @@ import org.labkey.api.attachments.AttachmentService; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DatabaseMigrationConfiguration; -import org.labkey.api.data.DatabaseMigrationService; -import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationSchemaHandler; -import org.labkey.api.data.DbSchema; import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.TableInfo; import org.labkey.api.module.CodeOnlyModule; import org.labkey.api.module.ModuleContext; import org.labkey.api.module.ModuleLoader; @@ -122,32 +117,6 @@ public void doStartup(ModuleContext moduleContext) WikiSchema.register(this); WikiController.registerAdminConsoleLinks(); - DatabaseMigrationService.get().registerSchemaHandler(new DefaultMigrationSchemaHandler(CommSchema.getInstance().getSchema()) - { - @Override - public void beforeSchema() - { - new SqlExecutor(getSchema()).execute("ALTER TABLE comm.Pages DROP CONSTRAINT FK_Pages_PageVersions"); - new SqlExecutor(getSchema()).execute("ALTER TABLE comm.Pages DROP CONSTRAINT FK_Pages_Parent"); - } - - @Override - public List getTablesToCopy() - { - List tablesToCopy = super.getTablesToCopy(); - tablesToCopy.add(CommSchema.getInstance().getTableInfoPages()); - tablesToCopy.add(CommSchema.getInstance().getTableInfoPageVersions()); - - return tablesToCopy; - } - - @Override - public void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema) - { - new SqlExecutor(getSchema()).execute("ALTER TABLE comm.Pages ADD CONSTRAINT FK_Pages_PageVersions FOREIGN KEY (PageVersionId) REFERENCES comm.PageVersions (RowId)"); - new SqlExecutor(getSchema()).execute("ALTER TABLE comm.Pages ADD CONSTRAINT FK_Pages_Parent FOREIGN KEY (Parent) REFERENCES comm.Pages (RowId)"); - } - }); } private void bootstrap(ModuleContext moduleContext) diff --git a/wiki/src/org/labkey/wiki/model/WikiType.java b/wiki/src/org/labkey/wiki/model/WikiType.java index 6ca8ec4b5e7..d45905f108f 100644 --- a/wiki/src/org/labkey/wiki/model/WikiType.java +++ b/wiki/src/org/labkey/wiki/model/WikiType.java @@ -16,6 +16,7 @@ package org.labkey.wiki.model; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.labkey.api.announcements.CommSchema; import org.labkey.api.attachments.AttachmentType; import org.labkey.api.data.SQLFragment; @@ -40,8 +41,8 @@ public static AttachmentType get() } @Override - public void addWhereSql(SQLFragment sql, String parentColumn, String documentNameColumn) + public @Nullable SQLFragment getSelectParentEntityIdsSql() { - sql.append(parentColumn).append(" IN (SELECT EntityId FROM ").append(CommSchema.getInstance().getTableInfoPages(), "pages").append(")"); + return new SQLFragment("SELECT EntityId FROM ").append(CommSchema.getInstance().getTableInfoPages(), "pages"); } } From 4ce9d7c3b56521b928199764fd9a6f8360cae9ea Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Fri, 7 Nov 2025 14:30:28 -0800 Subject: [PATCH 02/12] Eliminate TableHandler --- .../data/DatabaseMigrationConfiguration.java | 5 ++--- .../api/data/DatabaseMigrationService.java | 20 +++---------------- .../core/CoreMigrationSchemaHandler.java | 2 +- 3 files changed, 6 insertions(+), 21 deletions(-) diff --git a/api/src/org/labkey/api/data/DatabaseMigrationConfiguration.java b/api/src/org/labkey/api/data/DatabaseMigrationConfiguration.java index d6222ff90ac..d04e3b7abfe 100644 --- a/api/src/org/labkey/api/data/DatabaseMigrationConfiguration.java +++ b/api/src/org/labkey/api/data/DatabaseMigrationConfiguration.java @@ -3,7 +3,6 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.data.DatabaseMigrationService.MigrationSchemaHandler; -import org.labkey.api.data.DatabaseMigrationService.MigrationTableHandler; import java.util.Set; import java.util.function.Predicate; @@ -16,7 +15,7 @@ public interface DatabaseMigrationConfiguration DbScope getTargetScope(); @NotNull Set getSkipSchemas(); Predicate getColumnNameFilter(); - @Nullable TableSelector getTableSelector(DbSchemaType schemaType, TableInfo sourceTable, TableInfo targetTable, Set selectColumnNames, MigrationSchemaHandler schemaHandler, @Nullable MigrationTableHandler tableHandler); + @Nullable TableSelector getTableSelector(DbSchemaType schemaType, TableInfo sourceTable, TableInfo targetTable, Set selectColumnNames, MigrationSchemaHandler schemaHandler); class DefaultDatabaseMigrationConfiguration implements DatabaseMigrationConfiguration { @@ -51,7 +50,7 @@ public Predicate getColumnNameFilter() } @Override - public TableSelector getTableSelector(DbSchemaType schemaType, TableInfo sourceTable, TableInfo targetTable, Set selectColumnNames, MigrationSchemaHandler schemaHandler, @Nullable MigrationTableHandler tableHandler) + public TableSelector getTableSelector(DbSchemaType schemaType, TableInfo sourceTable, TableInfo targetTable, Set selectColumnNames, MigrationSchemaHandler schemaHandler) { return null; } diff --git a/api/src/org/labkey/api/data/DatabaseMigrationService.java b/api/src/org/labkey/api/data/DatabaseMigrationService.java index 466e4f3e30c..0eb3327a45d 100644 --- a/api/src/org/labkey/api/data/DatabaseMigrationService.java +++ b/api/src/org/labkey/api/data/DatabaseMigrationService.java @@ -59,7 +59,6 @@ default void migrate(DatabaseMigrationConfiguration configuration) // By default, no-op implementations default void registerSchemaHandler(MigrationSchemaHandler schemaHandler) {} - default void registerTableHandler(MigrationTableHandler tableHandler) {} default void registerMigrationFilter(MigrationFilter filter) {} default @Nullable MigrationFilter getMigrationFilter(String propertyName) @@ -309,7 +308,7 @@ public void afterTable(TableInfo sourceTable, TableInfo targetTable, SimpleFilte @Override public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema) { - // Now that the table tables in this schema have been populated, copy all associated attachments. By + // Now that the target tables in this schema have been populated, copy all associated attachments. By // default, use this handler's attachment types to select from the target tables all EntityIds that might be // attachment parents (this avoids re-running potentially expensive queries on the source tables). Use the // set of EntityIds to copy those attachments from the core.Documents table in the source database. Override @@ -326,10 +325,9 @@ public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSche // TODO: Mark this attachment type as having been seen // TODO: afterMigration() and update core.Documents' sequence + // TODO: implement a bunch more AttachmentTypes // TODO: throw if some registered AttachmentType is not seen - // TODO: get rid of TableHandler - // TODO: eventually, fail if type.getSelectParentEntityIdsSql() returns null - // TODO: implement a bunch more AttachmentTypes, override for Notebooks? + // TODO: fail if type.getSelectParentEntityIdsSql() returns null }); } @@ -359,18 +357,6 @@ public void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema s } } - /** - * Rarely needed, this interface allows a module to provide a clause that filters the rows of another module's - * table. The specific use case: Core manages core.Documents and LabBook implements its global attachment manager - * on top of core.Documents. When copying data from core.Documents, we want LabBook to filter out the rows that - * are not referenced by notebooks in the subset of containers being copied. - */ - interface MigrationTableHandler - { - TableInfo getTableInfo(); - FilterClause getAdditionalFilterClause(Set containers); - } - /** * A MigrationFilter adds support for the named filter property in the migration configuration file. If present, * saveFilter() is called with the container guid and property value. Modules can register these to present diff --git a/core/src/org/labkey/core/CoreMigrationSchemaHandler.java b/core/src/org/labkey/core/CoreMigrationSchemaHandler.java index 271ed39f5e5..bf4567dd40e 100644 --- a/core/src/org/labkey/core/CoreMigrationSchemaHandler.java +++ b/core/src/org/labkey/core/CoreMigrationSchemaHandler.java @@ -199,7 +199,7 @@ public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSche SQLFragment sql = new SQLFragment(); LookAndFeelResourceType.get().addWhereSql(sql, "Parent", "DocumentName"); - copyAttachments(configuration, sourceSchema, new SQLClause(sql), ""); + copyAttachments(configuration, sourceSchema, new SQLClause(sql), " associated with " + LookAndFeelResourceType.get().getClass().getSimpleName()); } @Override From 727b59bcdf7b74335dc955f2af5c6cbf1b60f87f Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Sat, 8 Nov 2025 08:38:17 -0800 Subject: [PATCH 03/12] Track attachment types that have been seen and report those not seen --- .../api/attachments/AttachmentService.java | 5 +++++ .../data/DatabaseMigrationConfiguration.java | 1 + .../api/data/DatabaseMigrationService.java | 20 +++++++++++++++---- .../api/data/TempTableInClauseGenerator.java | 2 -- .../core/CoreMigrationSchemaHandler.java | 5 ++--- .../attachment/AttachmentServiceImpl.java | 10 ++++++++-- 6 files changed, 32 insertions(+), 11 deletions(-) diff --git a/api/src/org/labkey/api/attachments/AttachmentService.java b/api/src/org/labkey/api/attachments/AttachmentService.java index 538d25674f9..01cd199f9de 100644 --- a/api/src/org/labkey/api/attachments/AttachmentService.java +++ b/api/src/org/labkey/api/attachments/AttachmentService.java @@ -133,6 +133,11 @@ static AttachmentService get() void registerAttachmentType(AttachmentType type); + /** + * Returns a collection of all registered AttachmentTypes + **/ + Collection getAttachmentTypes(); + HttpView getAdminView(ActionURL currentUrl); HttpView getFindAttachmentParentsView(); diff --git a/api/src/org/labkey/api/data/DatabaseMigrationConfiguration.java b/api/src/org/labkey/api/data/DatabaseMigrationConfiguration.java index d04e3b7abfe..26125f4c287 100644 --- a/api/src/org/labkey/api/data/DatabaseMigrationConfiguration.java +++ b/api/src/org/labkey/api/data/DatabaseMigrationConfiguration.java @@ -16,6 +16,7 @@ public interface DatabaseMigrationConfiguration @NotNull Set getSkipSchemas(); Predicate getColumnNameFilter(); @Nullable TableSelector getTableSelector(DbSchemaType schemaType, TableInfo sourceTable, TableInfo targetTable, Set selectColumnNames, MigrationSchemaHandler schemaHandler); + default void afterMigration(){}; class DefaultDatabaseMigrationConfiguration implements DatabaseMigrationConfiguration { diff --git a/api/src/org/labkey/api/data/DatabaseMigrationService.java b/api/src/org/labkey/api/data/DatabaseMigrationService.java index 0eb3327a45d..7e159828917 100644 --- a/api/src/org/labkey/api/data/DatabaseMigrationService.java +++ b/api/src/org/labkey/api/data/DatabaseMigrationService.java @@ -3,6 +3,7 @@ import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.labkey.api.attachments.AttachmentService; import org.labkey.api.attachments.AttachmentType; import org.labkey.api.data.DatabaseMigrationConfiguration.DefaultDatabaseMigrationConfiguration; import org.labkey.api.data.SimpleFilter.AndClause; @@ -320,10 +321,9 @@ public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSche { Collection entityIds = new SqlSelector(targetSchema, sql).getCollection(String.class); FilterClause filterClause = new InClause(FieldKey.fromParts("Parent"), entityIds); - copyAttachments(configuration, sourceSchema, filterClause, " associated with " + type.getClass().getSimpleName()); + copyAttachments(configuration, sourceSchema, filterClause, type); } - // TODO: Mark this attachment type as having been seen // TODO: afterMigration() and update core.Documents' sequence // TODO: implement a bunch more AttachmentTypes // TODO: throw if some registered AttachmentType is not seen @@ -331,12 +331,16 @@ public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSche }); } + private static final Set SEEN = new HashSet<>(); + // Copy all core.Documents rows that match the provided filter clause - protected void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, FilterClause filterClause, String additionalLogMessage) + protected void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, FilterClause filterClause, AttachmentType type) { + SEEN.add(type); + String additionalMessage = " associated with " + type.getClass().getSimpleName(); TableInfo sourceDocumentsTable = sourceSchema.getScope().getSchema("core", DbSchemaType.Migration).getTable("Documents"); TableInfo targetDocumentsTable = CoreSchema.getInstance().getTableInfoDocuments(); - DatabaseMigrationService.get().copySourceTableToTargetTable(configuration, sourceDocumentsTable, targetDocumentsTable, DbSchemaType.Module, false, additionalLogMessage, new DefaultMigrationSchemaHandler(CoreSchema.getInstance().getSchema()){ + DatabaseMigrationService.get().copySourceTableToTargetTable(configuration, sourceDocumentsTable, targetDocumentsTable, DbSchemaType.Module, false, additionalMessage, new DefaultMigrationSchemaHandler(CoreSchema.getInstance().getSchema()){ @Override public FilterClause getTableFilterClause(TableInfo sourceTable, Set containers) { @@ -345,6 +349,14 @@ public FilterClause getTableFilterClause(TableInfo sourceTable, Set contai }); } + public static void logUnseenAttachmentTypes() + { + Set unseen = new HashSet<>(AttachmentService.get().getAttachmentTypes()); + unseen.removeAll(SEEN); + + LOG.info("These AttachmentTypes have not been seen: {}", unseen.stream().map(type -> type.getClass().getSimpleName()).collect(Collectors.joining(", "))); + } + @Override public @NotNull Collection getAttachmentTypes() { diff --git a/api/src/org/labkey/api/data/TempTableInClauseGenerator.java b/api/src/org/labkey/api/data/TempTableInClauseGenerator.java index 7876a83fae6..6ccaa4d5009 100644 --- a/api/src/org/labkey/api/data/TempTableInClauseGenerator.java +++ b/api/src/org/labkey/api/data/TempTableInClauseGenerator.java @@ -35,8 +35,6 @@ import java.util.TreeSet; /** - * Created by davebradlee on 6/5/15. - * * Generator for very long in-clauses */ public class TempTableInClauseGenerator implements InClauseGenerator diff --git a/core/src/org/labkey/core/CoreMigrationSchemaHandler.java b/core/src/org/labkey/core/CoreMigrationSchemaHandler.java index bf4567dd40e..b2248744dac 100644 --- a/core/src/org/labkey/core/CoreMigrationSchemaHandler.java +++ b/core/src/org/labkey/core/CoreMigrationSchemaHandler.java @@ -192,14 +192,13 @@ public void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema s @Override public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema) { - // Default handling for most of the attachment types + // Default handling for core's standard attachment types super.copyAttachments(configuration, sourceSchema, targetSchema); // Special handling for LookAndFeelResourceType, which must select from the source database SQLFragment sql = new SQLFragment(); LookAndFeelResourceType.get().addWhereSql(sql, "Parent", "DocumentName"); - - copyAttachments(configuration, sourceSchema, new SQLClause(sql), " associated with " + LookAndFeelResourceType.get().getClass().getSimpleName()); + copyAttachments(configuration, sourceSchema, new SQLClause(sql), LookAndFeelResourceType.get()); } @Override diff --git a/core/src/org/labkey/core/attachment/AttachmentServiceImpl.java b/core/src/org/labkey/core/attachment/AttachmentServiceImpl.java index 9755d824801..1aa5400126b 100644 --- a/core/src/org/labkey/core/attachment/AttachmentServiceImpl.java +++ b/core/src/org/labkey/core/attachment/AttachmentServiceImpl.java @@ -747,6 +747,12 @@ public void registerAttachmentType(AttachmentType type) ATTACHMENT_TYPE_MAP.put(type.getUniqueName(), type); } + @Override + public Collection getAttachmentTypes() + { + return ATTACHMENT_TYPE_MAP.values(); + } + @Override public HttpView getAdminView(ActionURL currentUrl) { @@ -761,7 +767,7 @@ public HttpView getAdminView(ActionURL currentUrl) // core.Documents for each type is needed to associate the Type values with the associated rows. List selectStatements = new LinkedList<>(); - for (AttachmentType type : ATTACHMENT_TYPE_MAP.values()) + for (AttachmentType type : getAttachmentTypes()) { SQLFragment selectStatement = new SQLFragment(); @@ -785,7 +791,7 @@ public HttpView getAdminView(ActionURL currentUrl) SQLFragment whereSql = new SQLFragment(); String sep = ""; - for (AttachmentType type : ATTACHMENT_TYPE_MAP.values()) + for (AttachmentType type : getAttachmentTypes()) { whereSql.append(sep); sep = " OR"; From 6ce582e2c6a2236fe22caf3073b0f03db2b660b6 Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Sat, 8 Nov 2025 16:43:38 -0800 Subject: [PATCH 04/12] Teach TempTableInClauseGenerator how to target a temp table in an external data source. Adjust the attachment copying code to use this new mechanism. --- .../api/data/DatabaseMigrationService.java | 14 ++++++- .../labkey/api/data/InClauseGenerator.java | 2 - .../api/data/TempTableInClauseGenerator.java | 40 ++++++++++++++----- .../org/labkey/api/data/TempTableInfo.java | 17 ++------ .../data/dialect/BasePostgreSqlDialect.java | 8 +++- .../labkey/api/data/dialect/SqlDialect.java | 17 +++++--- 6 files changed, 63 insertions(+), 35 deletions(-) diff --git a/api/src/org/labkey/api/data/DatabaseMigrationService.java b/api/src/org/labkey/api/data/DatabaseMigrationService.java index 7e159828917..9e53d935c82 100644 --- a/api/src/org/labkey/api/data/DatabaseMigrationService.java +++ b/api/src/org/labkey/api/data/DatabaseMigrationService.java @@ -320,10 +320,13 @@ public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSche if (sql != null) { Collection entityIds = new SqlSelector(targetSchema, sql).getCollection(String.class); - FilterClause filterClause = new InClause(FieldKey.fromParts("Parent"), entityIds); - copyAttachments(configuration, sourceSchema, filterClause, type); + SQLFragment selectParents = new SQLFragment("Parent"); + // This query against the source database is likely to contain a large IN clause, so use an alternative InClauseGenerator + sourceSchema.getSqlDialect().appendInClauseSql(selectParents, entityIds, getTempTableInClauseGenerator(sourceSchema.getScope())); + copyAttachments(configuration, sourceSchema, new SQLClause(selectParents), type); } + // TODO: Fix & test issues attachment copy - need to invoke after provisioned table // TODO: afterMigration() and update core.Documents' sequence // TODO: implement a bunch more AttachmentTypes // TODO: throw if some registered AttachmentType is not seen @@ -331,6 +334,13 @@ public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSche }); } + // Creates an TempTableInClauseGenerator that targets the *source* temp schema instead of the default + // DbSchema.getTemp(). Required for large IN clauses against the source database. + protected InClauseGenerator getTempTableInClauseGenerator(DbScope sourceScope) + { + return new TempTableInClauseGenerator(() -> sourceScope.getSchema("temp", DbSchemaType.Bare)); + } + private static final Set SEEN = new HashSet<>(); // Copy all core.Documents rows that match the provided filter clause diff --git a/api/src/org/labkey/api/data/InClauseGenerator.java b/api/src/org/labkey/api/data/InClauseGenerator.java index ec5a60acdb4..0ee8fdb56a6 100644 --- a/api/src/org/labkey/api/data/InClauseGenerator.java +++ b/api/src/org/labkey/api/data/InClauseGenerator.java @@ -22,8 +22,6 @@ /** * Implementors generate and append SQL that acts as an "is one of" filter. This can be an actual IN clause or a * database-specific implementation that scales or performs better (e.g., arrays or in-line parameter expansion) - * User: adam - * Date: 8/3/12 */ public interface InClauseGenerator { diff --git a/api/src/org/labkey/api/data/TempTableInClauseGenerator.java b/api/src/org/labkey/api/data/TempTableInClauseGenerator.java index 6ccaa4d5009..14276fb551d 100644 --- a/api/src/org/labkey/api/data/TempTableInClauseGenerator.java +++ b/api/src/org/labkey/api/data/TempTableInClauseGenerator.java @@ -33,6 +33,7 @@ import java.util.List; import java.util.Set; import java.util.TreeSet; +import java.util.function.Supplier; /** * Generator for very long in-clauses @@ -42,6 +43,22 @@ public class TempTableInClauseGenerator implements InClauseGenerator private static final Cache _tempTableCache = CacheManager.getStringKeyCache(200, CacheManager.MINUTE * 5, "IN clause temp tables"); + // Need to set a supplier instead of setting the default temp schema directly because this class is constructed at + // dialect init time, before schemas can be referenced. + private final Supplier _tempSchemaSupplier; + + // By default, use the primary database temp schema + public TempTableInClauseGenerator() + { + this(DbSchema::getTemp); + } + + // Use in cases where the default temp schema won't do, e.g., you need to apply a large IN clause in an external data source + public TempTableInClauseGenerator(Supplier tempSchemaSupplier) + { + _tempSchemaSupplier = tempSchemaSupplier; + } + /** * @param sql fragment to append to * @param params list of values @@ -85,19 +102,20 @@ else if (jdbcType == JdbcType.VARCHAR) TempTableInfo tempTableInfo = _tempTableCache.get(cacheKey); if (tempTableInfo == null) { - tempTableInfo = new TempTableInfo("InClause", Collections.singletonList(new BaseColumnInfo("Id", jdbcType, 0, false)), null); + DbSchema tempSchema = _tempSchemaSupplier.get(); + tempTableInfo = new TempTableInfo(tempSchema, "InClause", Collections.singletonList(new BaseColumnInfo("Id", jdbcType, 0, false)), null); SQLFragment sqlCreate = new SQLFragment("CREATE TABLE "); sqlCreate.append(tempTableInfo) - .append("\n(Id ") - .append(DbSchema.getTemp().getSqlDialect().getSqlTypeName(jdbcType)) - .append(jdbcType == JdbcType.VARCHAR ? "(450)" : "") - .append(")"); + .append("\n(Id ") + .append(tempSchema.getSqlDialect().getSqlTypeName(jdbcType)) + .append(jdbcType == JdbcType.VARCHAR ? "(450)" : "") + .append(")"); // When the in clause receives more parameters than it is set to handle, a temporary table is created to handle the overflow. // While the associated mutating operations are necessary, they are not a viable CSRF attack vector. try (var ignored = SpringActionController.ignoreSqlUpdates()) { - new SqlExecutor(DbSchema.getTemp()).execute(sqlCreate); + new SqlExecutor(tempSchema).execute(sqlCreate); } tempTableInfo.track(); String tableName = tempTableInfo.getSelectName(); @@ -108,11 +126,11 @@ else if (jdbcType == JdbcType.VARCHAR) try (var ignored = SpringActionController.ignoreSqlUpdates()) { if (jdbcType == JdbcType.VARCHAR) - Table.batchExecute1String(DbSchema.getTemp(), sql1, (ArrayList) sortedParameters); + Table.batchExecute1String(tempSchema, sql1, (ArrayList) sortedParameters); else if (jdbcType == JdbcType.INTEGER) - Table.batchExecute1Integer(DbSchema.getTemp(), sql1, sql100, (ArrayList) sortedParameters); + Table.batchExecute1Integer(tempSchema, sql1, sql100, (ArrayList) sortedParameters); else - Table.batchExecute1Long(DbSchema.getTemp(), sql1, sql100, (ArrayList) sortedParameters); + Table.batchExecute1Long(tempSchema, sql1, sql100, (ArrayList) sortedParameters); } } catch (SQLException e) @@ -123,14 +141,14 @@ else if (jdbcType == JdbcType.INTEGER) String indexSql = "CREATE INDEX IX_Id" + new GUID().toStringNoDashes() + " ON " + tableName + "(Id)"; try (var ignored = SpringActionController.ignoreSqlUpdates()) { - new SqlExecutor(DbSchema.getTemp()).execute(indexSql); + new SqlExecutor(tempSchema).execute(indexSql); } TempTableInfo cacheEntry = tempTableInfo; // Don't bother caching if we're in a transaction // a) The table won't be visible to other connections until we commit // b) It is more likely that this temptable is only used once anyway (e.g. used by a data iterator) - if (!DbSchema.getTemp().getScope().isTransactionActive()) + if (!tempSchema.getScope().isTransactionActive()) _tempTableCache.put(cacheKey, cacheEntry); } diff --git a/api/src/org/labkey/api/data/TempTableInfo.java b/api/src/org/labkey/api/data/TempTableInfo.java index 21b56de3341..dbc6c63cd50 100644 --- a/api/src/org/labkey/api/data/TempTableInfo.java +++ b/api/src/org/labkey/api/data/TempTableInfo.java @@ -19,15 +19,8 @@ import java.util.List; -/** -* User: matt -* Date: Oct 23, 2010 -* Time: 3:08:13 PM -*/ public class TempTableInfo extends SchemaTableInfo { - private final String _tempTableName; - private TempTableTracker _ttt; public TempTableInfo(String name, List cols, List pk) @@ -35,7 +28,7 @@ public TempTableInfo(String name, List cols, List pk) this(DbSchema.getTemp(), name, cols, pk); } - private TempTableInfo(DbSchema schema, String name, List cols, List pk) + public TempTableInfo(DbSchema schema, String name, List cols, List pk) { super(schema, DatabaseTableType.TABLE, name, name, new SQLFragment().appendIdentifier(schema.getName()).append(".").appendIdentifier(name + "$" + new GUID().toStringNoDashes())); @@ -43,9 +36,6 @@ private TempTableInfo(DbSchema schema, String name, List cols, List< // make sure TempTableTracker is initialized _before_ caller executes CREATE TABLE TempTableTracker.init(); - // TODO: Do away with _tempTableName? getSelectName() is synonymous. - _tempTableName = getSelectName(); - for (var col : cols) { ((BaseColumnInfo)col).setParentTable(this); @@ -58,15 +48,14 @@ private TempTableInfo(DbSchema schema, String name, List cols, List< public String getTempTableName() { - return _tempTableName; + return getSelectName(); } - /** Call this method when table is physically created */ public void track() { // Remove the schema name and dot - String tableName = _tempTableName.substring(getSchema().getName().length() + 1); + String tableName = getTempTableName().substring(getSchema().getName().length() + 1); _ttt = TempTableTracker.track(tableName, this); } diff --git a/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java b/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java index 4d297f83581..5cba2821b96 100644 --- a/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java +++ b/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java @@ -291,10 +291,16 @@ public String addReselect(SQLFragment sql, ColumnInfo column, @Nullable String p @Override public SQLFragment appendInClauseSql(SQLFragment sql, @NotNull Collection params) + { + return appendInClauseSql(sql, params, _tempTableInClauseGenerator); + } + + @Override + public SQLFragment appendInClauseSql(SQLFragment sql, @NotNull Collection params, InClauseGenerator tempTableGenerator) { if (params.size() >= TEMPTABLE_GENERATOR_MINSIZE) { - SQLFragment ret = _tempTableInClauseGenerator.appendInClauseSql(sql, params); + SQLFragment ret = tempTableGenerator.appendInClauseSql(sql, params); if (null != ret) return ret; } diff --git a/api/src/org/labkey/api/data/dialect/SqlDialect.java b/api/src/org/labkey/api/data/dialect/SqlDialect.java index 0edb407cf39..5c19524f10c 100644 --- a/api/src/org/labkey/api/data/dialect/SqlDialect.java +++ b/api/src/org/labkey/api/data/dialect/SqlDialect.java @@ -516,7 +516,7 @@ protected Set getJdbcKeywords(SqlExecutor executor) throws SQLException, * @param sql And INSERT or UPDATE statement that needs re-selecting * @param column Column from which to reselect * @param proposedVariable Null to return a result set via code; Not null to select the value into a SQL variable - * @return If proposedVariable is not null then actual variable used in the SQL. Otherwise null. Callers using + * @return If proposedVariable is not null then actual variable used in the SQL. Otherwise, null. Callers using * proposedVariable must use the returned variable name in subsequent code, since it may differ from what was * proposed. */ @@ -527,7 +527,14 @@ protected Set getJdbcKeywords(SqlExecutor executor) throws SQLException, private static final InClauseGenerator DEFAULT_GENERATOR = new ParameterMarkerInClauseGenerator(); + // Most callers should use this method public SQLFragment appendInClauseSql(SQLFragment sql, @NotNull Collection params) + { + return appendInClauseSql(sql, params, null); + } + + // Use in cases where the default temp schema won't do, e.g., you need to apply a large IN clause in an external data source + public SQLFragment appendInClauseSql(SQLFragment sql, @NotNull Collection params, InClauseGenerator tempTableGenerator) { return DEFAULT_GENERATOR.appendInClauseSql(sql, params); } @@ -539,10 +546,10 @@ public SQLFragment appendCaseInsensitiveLikeClause(SQLFragment sql, @NotNull Str String prefixLike = prefix + CompareType.escapeLikePattern(matchStr, escapeChar) + suffix; String escapeToken = " ESCAPE '" + escapeChar + "'"; sql.append(" ") - .append(getCaseInsensitiveLikeOperator()) - .append(" ") - .appendValue(prefixLike) - .append(escapeToken); + .append(getCaseInsensitiveLikeOperator()) + .append(" ") + .appendValue(prefixLike) + .append(escapeToken); return sql; } From ac97789a859b610d959f7fd3ed5061b1c875996e Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Sun, 9 Nov 2025 10:57:33 -0800 Subject: [PATCH 05/12] Handle attachment migration for more schemas --- .../announcements/AnnouncementModule.java | 8 ++-- api/src/org/labkey/api/ApiModule.java | 2 - .../api/attachments/SecureDocumentType.java | 45 ------------------- .../api/data/DatabaseMigrationService.java | 15 ++++--- api/src/org/labkey/api/wiki/WikiService.java | 3 ++ .../issue/IssueMigrationSchemaHandler.java | 39 +++++++--------- study/src/org/labkey/study/StudyModule.java | 7 +++ .../study/model/ProtocolDocumentType.java | 5 ++- wiki/src/org/labkey/wiki/WikiManager.java | 9 +++- 9 files changed, 49 insertions(+), 84 deletions(-) delete mode 100644 api/src/org/labkey/api/attachments/SecureDocumentType.java diff --git a/announcements/src/org/labkey/announcements/AnnouncementModule.java b/announcements/src/org/labkey/announcements/AnnouncementModule.java index 0479fa98961..6125aa11f12 100644 --- a/announcements/src/org/labkey/announcements/AnnouncementModule.java +++ b/announcements/src/org/labkey/announcements/AnnouncementModule.java @@ -59,6 +59,7 @@ import org.labkey.api.view.ViewContext; import org.labkey.api.view.WebPartFactory; import org.labkey.api.view.WebPartView; +import org.labkey.api.wiki.WikiService; import java.util.ArrayList; import java.util.Collection; @@ -202,10 +203,9 @@ public void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema s @Override public @NotNull Collection getAttachmentTypes() { - // TODO: Need a way to get WikiType in here - return List.of( - AnnouncementType.get() - ); + // It's theoretically possible to deploy Announcement without Wiki, so conditionalize + WikiService ws = WikiService.get(); + return ws != null ? List.of(AnnouncementType.get(), ws.getAttachmentType()) : List.of(AnnouncementType.get()); } }); } diff --git a/api/src/org/labkey/api/ApiModule.java b/api/src/org/labkey/api/ApiModule.java index a14f28c3fef..afd068e7a0e 100644 --- a/api/src/org/labkey/api/ApiModule.java +++ b/api/src/org/labkey/api/ApiModule.java @@ -31,7 +31,6 @@ import org.labkey.api.attachments.AttachmentService; import org.labkey.api.attachments.ImageServlet; import org.labkey.api.attachments.LookAndFeelResourceType; -import org.labkey.api.attachments.SecureDocumentType; import org.labkey.api.audit.query.AbstractAuditDomainKind; import org.labkey.api.cache.BlockingCache; import org.labkey.api.collections.ArrayListMap; @@ -222,7 +221,6 @@ protected void init() AttachmentService.get().registerAttachmentType(LookAndFeelResourceType.get()); AttachmentService.get().registerAttachmentType(AuthenticationLogoType.get()); AttachmentService.get().registerAttachmentType(AvatarType.get()); - AttachmentService.get().registerAttachmentType(SecureDocumentType.get()); PropertyManager.registerEncryptionMigrationHandler(); AuthenticationManager.registerEncryptionMigrationHandler(); diff --git a/api/src/org/labkey/api/attachments/SecureDocumentType.java b/api/src/org/labkey/api/attachments/SecureDocumentType.java deleted file mode 100644 index 5146449c416..00000000000 --- a/api/src/org/labkey/api/attachments/SecureDocumentType.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2017 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.attachments; - -import org.jetbrains.annotations.NotNull; -import org.labkey.api.data.SQLFragment; - -public class SecureDocumentType implements AttachmentType -{ - private static final SecureDocumentType INSTANCE = new SecureDocumentType(); - - public static SecureDocumentType get() - { - return INSTANCE; - } - - private SecureDocumentType() - { - } - - @Override - public @NotNull String getUniqueName() - { - return getClass().getName(); - } - - @Override - public void addWhereSql(SQLFragment sql, String parentColumn, String documentNameColumn) - { - sql.append("1 = 0"); // No secure documents in current deployments - } -} \ No newline at end of file diff --git a/api/src/org/labkey/api/data/DatabaseMigrationService.java b/api/src/org/labkey/api/data/DatabaseMigrationService.java index 9e53d935c82..66754be885f 100644 --- a/api/src/org/labkey/api/data/DatabaseMigrationService.java +++ b/api/src/org/labkey/api/data/DatabaseMigrationService.java @@ -22,6 +22,7 @@ import org.labkey.vfs.FileLike; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -326,16 +327,16 @@ public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSche copyAttachments(configuration, sourceSchema, new SQLClause(selectParents), type); } - // TODO: Fix & test issues attachment copy - need to invoke after provisioned table + // TODO: **Test issues attachment copy + // TODO: implement remaining AttachmentTypes // TODO: afterMigration() and update core.Documents' sequence - // TODO: implement a bunch more AttachmentTypes // TODO: throw if some registered AttachmentType is not seen // TODO: fail if type.getSelectParentEntityIdsSql() returns null }); } - // Creates an TempTableInClauseGenerator that targets the *source* temp schema instead of the default - // DbSchema.getTemp(). Required for large IN clauses against the source database. + // Creates a TempTableInClauseGenerator that targets the *source* temp schema instead of the default + // DbSchema.getTemp(). Required for large IN clauses used against the source database. protected InClauseGenerator getTempTableInClauseGenerator(DbScope sourceScope) { return new TempTableInClauseGenerator(() -> sourceScope.getSchema("temp", DbSchemaType.Bare)); @@ -344,10 +345,10 @@ protected InClauseGenerator getTempTableInClauseGenerator(DbScope sourceScope) private static final Set SEEN = new HashSet<>(); // Copy all core.Documents rows that match the provided filter clause - protected void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, FilterClause filterClause, AttachmentType type) + protected void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, FilterClause filterClause, AttachmentType... type) { - SEEN.add(type); - String additionalMessage = " associated with " + type.getClass().getSimpleName(); + SEEN.addAll(Arrays.asList(type)); + String additionalMessage = " associated with " + Arrays.stream(type).map(t -> t.getClass().getSimpleName()).collect(Collectors.joining(", ")); TableInfo sourceDocumentsTable = sourceSchema.getScope().getSchema("core", DbSchemaType.Migration).getTable("Documents"); TableInfo targetDocumentsTable = CoreSchema.getInstance().getTableInfoDocuments(); DatabaseMigrationService.get().copySourceTableToTargetTable(configuration, sourceDocumentsTable, targetDocumentsTable, DbSchemaType.Module, false, additionalMessage, new DefaultMigrationSchemaHandler(CoreSchema.getInstance().getSchema()){ diff --git a/api/src/org/labkey/api/wiki/WikiService.java b/api/src/org/labkey/api/wiki/WikiService.java index dfae120d75b..1c0f2afc7a6 100644 --- a/api/src/org/labkey/api/wiki/WikiService.java +++ b/api/src/org/labkey/api/wiki/WikiService.java @@ -19,6 +19,7 @@ import org.jetbrains.annotations.Nullable; import org.labkey.api.attachments.AttachmentFile; import org.labkey.api.attachments.AttachmentParent; +import org.labkey.api.attachments.AttachmentType; import org.labkey.api.data.Container; import org.labkey.api.data.TableInfo; import org.labkey.api.security.User; @@ -94,4 +95,6 @@ static void setInstance(WikiService impl) */ @Nullable String updateAttachments(Container c, User user, String wikiName, @Nullable List attachmentFiles, @Nullable List deleteAttachmentNames); + + AttachmentType getAttachmentType(); } diff --git a/issues/src/org/labkey/issue/IssueMigrationSchemaHandler.java b/issues/src/org/labkey/issue/IssueMigrationSchemaHandler.java index 0efa2667ee9..523ec648013 100644 --- a/issues/src/org/labkey/issue/IssueMigrationSchemaHandler.java +++ b/issues/src/org/labkey/issue/IssueMigrationSchemaHandler.java @@ -31,7 +31,7 @@ public class IssueMigrationSchemaHandler extends DefaultMigrationSchemaHandler { private static final Logger LOG = LogHelper.getLogger(IssueMigrationSchemaHandler.class, "Issue migration status"); - private final Set ISSUE_IDS = new HashSet<>(); + private final Set COPIED_ISSUE_IDS = new HashSet<>(); public IssueMigrationSchemaHandler() { @@ -43,7 +43,7 @@ public void afterTable(TableInfo sourceTable, TableInfo targetTable, SimpleFilte { // Collect the issue IDs that were copied into the target table. We're assuming this set is much smaller than // the set of issues IDs that *weren't* copied. - int startSize = ISSUE_IDS.size(); + int startSize = COPIED_ISSUE_IDS.size(); // Join the provisioned table to the issues table to get the IssueIds associated with the rows that were copied SQLClause joinOnEntityId = new SQLClause( @@ -53,33 +53,26 @@ public void afterTable(TableInfo sourceTable, TableInfo targetTable, SimpleFilte ); new TableSelector(IssuesSchema.getInstance().getTableInfoIssues(), new CsvSet("IssueId, EntityId"), new SimpleFilter(joinOnEntityId), null).stream(Integer.class) - .forEach(ISSUE_IDS::add); - LOG.info(" {} added to the IssueId set", StringUtilsLabKey.pluralize(ISSUE_IDS.size() - startSize, "IssueId was", "IssueIds were")); + .forEach(COPIED_ISSUE_IDS::add); + LOG.info(" {} added to the IssueId set", StringUtilsLabKey.pluralize(COPIED_ISSUE_IDS.size() - startSize, "IssueId was", "IssueIds were")); } @Override public void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema) { - LOG.info(" Deleting related issues, comments, and issues rows associated with {}", StringUtilsLabKey.pluralize(ISSUE_IDS.size(), "issue")); + LOG.info("{} were copied. Now deleting related issues, comments, and issues rows associated with all issues that were not copied.", StringUtilsLabKey.pluralize(COPIED_ISSUE_IDS.size(), "issue")); - if (!ISSUE_IDS.isEmpty()) - { - // Delete all issues, comments, and related issues that were NOT copied - SimpleFilter deleteRelatedFilter = new SimpleFilter( - new NotClause( - new InClause(FieldKey.fromParts("RelatedIssueId"), ISSUE_IDS) - ) - ); - Table.delete(IssuesSchema.getInstance().getTableInfoRelatedIssues(), deleteRelatedFilter); - SimpleFilter deleteFilter = new SimpleFilter( - new NotClause( - new InClause(FieldKey.fromParts("IssueId"), ISSUE_IDS) - ) - ); - Table.delete(IssuesSchema.getInstance().getTableInfoRelatedIssues(), deleteFilter); - Table.delete(IssuesSchema.getInstance().getTableInfoComments(), deleteFilter); - Table.delete(IssuesSchema.getInstance().getTableInfoIssues(), deleteFilter); - } + // Delete all issues, comments, and related issues that were NOT copied + SimpleFilter deleteRelatedFilter = new SimpleFilter( + new InClause(FieldKey.fromParts("RelatedIssueId"), COPIED_ISSUE_IDS, false, true) // Negated + ); + Table.delete(IssuesSchema.getInstance().getTableInfoRelatedIssues(), deleteRelatedFilter); + SimpleFilter deleteFilter = new SimpleFilter( + new InClause(FieldKey.fromParts("IssueId"), COPIED_ISSUE_IDS, false, true) // Negated + ); + Table.delete(IssuesSchema.getInstance().getTableInfoRelatedIssues(), deleteFilter); + Table.delete(IssuesSchema.getInstance().getTableInfoComments(), deleteFilter); + Table.delete(IssuesSchema.getInstance().getTableInfoIssues(), deleteFilter); } @Override diff --git a/study/src/org/labkey/study/StudyModule.java b/study/src/org/labkey/study/StudyModule.java index 63b2e9f25b8..9147499cbfa 100644 --- a/study/src/org/labkey/study/StudyModule.java +++ b/study/src/org/labkey/study/StudyModule.java @@ -25,6 +25,7 @@ import org.labkey.api.admin.FolderSerializationRegistry; import org.labkey.api.admin.notification.NotificationService; import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.attachments.AttachmentType; import org.labkey.api.audit.AuditLogService; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; @@ -537,6 +538,12 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM { return "StudySnapshot".equals(sourceTable.getName()) ? FieldKey.fromParts("Source") : super.getContainerFieldKey(sourceTable); } + + @Override + public @NotNull Collection getAttachmentTypes() + { + return List.of(ProtocolDocumentType.get()); + } }); DatabaseMigrationService.get().registerSchemaHandler(new DefaultMigrationSchemaHandler(StudySchema.getInstance().getDatasetSchema()) diff --git a/study/src/org/labkey/study/model/ProtocolDocumentType.java b/study/src/org/labkey/study/model/ProtocolDocumentType.java index 88f08432613..25b25aa616b 100644 --- a/study/src/org/labkey/study/model/ProtocolDocumentType.java +++ b/study/src/org/labkey/study/model/ProtocolDocumentType.java @@ -16,6 +16,7 @@ package org.labkey.study.model; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.labkey.api.attachments.AttachmentType; import org.labkey.api.data.SQLFragment; import org.labkey.study.StudySchema; @@ -40,8 +41,8 @@ private ProtocolDocumentType() } @Override - public void addWhereSql(SQLFragment sql, String parentColumn, String documentNameColumn) + public @Nullable SQLFragment getSelectParentEntityIdsSql() { - sql.append(parentColumn).append(" IN (SELECT ProtocolDocumentEntityId FROM ").append(StudySchema.getInstance().getTableInfoStudy(), "s").append(")"); + return new SQLFragment("SELECT ProtocolDocumentEntityId FROM ").append(StudySchema.getInstance().getTableInfoStudy(), "s"); } } diff --git a/wiki/src/org/labkey/wiki/WikiManager.java b/wiki/src/org/labkey/wiki/WikiManager.java index 4cc65405268..fafd3cd108f 100644 --- a/wiki/src/org/labkey/wiki/WikiManager.java +++ b/wiki/src/org/labkey/wiki/WikiManager.java @@ -29,6 +29,7 @@ import org.labkey.api.attachments.AttachmentParent; import org.labkey.api.attachments.AttachmentService; import org.labkey.api.attachments.AttachmentService.DuplicateFilenameException; +import org.labkey.api.attachments.AttachmentType; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerService; import org.labkey.api.data.CoreSchema; @@ -70,6 +71,7 @@ import org.labkey.api.wiki.WikiRenderingService.SubstitutionMode; import org.labkey.api.wiki.WikiService; import org.labkey.wiki.model.Wiki; +import org.labkey.wiki.model.WikiType; import org.labkey.wiki.model.WikiVersion; import org.labkey.wiki.model.WikiVersionsGrid; import org.labkey.wiki.model.WikiView; @@ -1056,6 +1058,12 @@ public void deleteWiki(Container c, User user, String wikiName, boolean deleteSu return null; } + @Override + public AttachmentType getAttachmentType() + { + return WikiType.get(); + } + public static class TestCase extends Assert { WikiManager _m = null; @@ -1074,7 +1082,6 @@ public void testSchema() assertNotNull(_m.comm.getTableInfoPages().getColumn("EntityId")); assertNotNull(_m.comm.getTableInfoPages().getColumn("Name")); - assertNotNull("couldn't find table PageVersions", _m.comm.getTableInfoPageVersions()); assertNotNull(_m.comm.getTableInfoPageVersions().getColumn("PageEntityId")); assertNotNull(_m.comm.getTableInfoPageVersions().getColumn("Title")); From 6ce4724a29547963f98e40b70bd167892d2e2f4e Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Mon, 10 Nov 2025 13:48:23 -0800 Subject: [PATCH 06/12] Log deleted issue rows. Implement attachment handling for lists and specimen requests. --- api/src/org/labkey/api/search/SearchService.java | 4 ++-- api/src/org/labkey/api/study/SpecimenService.java | 3 +++ .../labkey/issue/IssueMigrationSchemaHandler.java | 13 ++++++++----- list/src/org/labkey/list/ListModule.java | 11 +++++++++++ list/src/org/labkey/list/view/ListItemType.java | 11 ++++++++--- .../search/model/LuceneSearchServiceImpl.java | 2 +- .../org/labkey/specimen/SpecimenServiceImpl.java | 8 ++++++++ .../specimen/model/SpecimenRequestEventType.java | 5 +++-- study/src/org/labkey/study/StudyModule.java | 9 ++++++++- 9 files changed, 52 insertions(+), 14 deletions(-) diff --git a/api/src/org/labkey/api/search/SearchService.java b/api/src/org/labkey/api/search/SearchService.java index 5b09af23b83..4d64b9ce747 100644 --- a/api/src/org/labkey/api/search/SearchService.java +++ b/api/src/org/labkey/api/search/SearchService.java @@ -394,7 +394,7 @@ public String normalizeHref(Path contextPath, Container c) DbSchema getSchema(); - WebPartView getSearchView(boolean includeSubfolders, int textBoxWidth, boolean includeHelpLink, boolean isWebpart); + WebPartView getSearchView(boolean includeSubfolders, int textBoxWidth, boolean includeHelpLink, boolean isWebpart); SearchResult search(SearchOptions options) throws IOException; @@ -462,7 +462,7 @@ public String normalizeHref(Path contextPath, Container c) void addResourceResolver(@NotNull String prefix, @NotNull ResourceResolver resolver); WebdavResource resolveResource(@NotNull String resourceIdentifier); - HttpView getCustomSearchResult(User user, @NotNull String resourceIdentifier); + HttpView getCustomSearchResult(User user, @NotNull String resourceIdentifier); Map getCustomSearchJson(User user, @NotNull String resourceIdentifier); Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers); diff --git a/api/src/org/labkey/api/study/SpecimenService.java b/api/src/org/labkey/api/study/SpecimenService.java index f7b197d30af..e56b3f55f09 100644 --- a/api/src/org/labkey/api/study/SpecimenService.java +++ b/api/src/org/labkey/api/study/SpecimenService.java @@ -20,6 +20,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.annotations.Migrate; +import org.labkey.api.attachments.AttachmentType; import org.labkey.api.data.Container; import org.labkey.api.data.TableInfo; import org.labkey.api.exp.Lsid; @@ -93,6 +94,8 @@ static SpecimenService get() void registerRequestCustomizer(SpecimenRequestCustomizer customizer); + AttachmentType getSpecimenRequestEventType(); + /** Hooks to allow other modules to control a few items about how specimens are treated */ interface SpecimenRequestCustomizer { diff --git a/issues/src/org/labkey/issue/IssueMigrationSchemaHandler.java b/issues/src/org/labkey/issue/IssueMigrationSchemaHandler.java index 523ec648013..ee5f8d48eb6 100644 --- a/issues/src/org/labkey/issue/IssueMigrationSchemaHandler.java +++ b/issues/src/org/labkey/issue/IssueMigrationSchemaHandler.java @@ -11,7 +11,6 @@ import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.SimpleFilter.InClause; -import org.labkey.api.data.SimpleFilter.NotClause; import org.labkey.api.data.SimpleFilter.SQLClause; import org.labkey.api.data.Table; import org.labkey.api.data.TableInfo; @@ -66,13 +65,17 @@ public void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema s SimpleFilter deleteRelatedFilter = new SimpleFilter( new InClause(FieldKey.fromParts("RelatedIssueId"), COPIED_ISSUE_IDS, false, true) // Negated ); - Table.delete(IssuesSchema.getInstance().getTableInfoRelatedIssues(), deleteRelatedFilter); + int deletedRowCount = Table.delete(IssuesSchema.getInstance().getTableInfoRelatedIssues(), deleteRelatedFilter); + LOG.info(" Deleted {} from RelatedIssues (RelatedIssueId)", StringUtilsLabKey.pluralize(deletedRowCount, "row")); SimpleFilter deleteFilter = new SimpleFilter( new InClause(FieldKey.fromParts("IssueId"), COPIED_ISSUE_IDS, false, true) // Negated ); - Table.delete(IssuesSchema.getInstance().getTableInfoRelatedIssues(), deleteFilter); - Table.delete(IssuesSchema.getInstance().getTableInfoComments(), deleteFilter); - Table.delete(IssuesSchema.getInstance().getTableInfoIssues(), deleteFilter); + deletedRowCount = Table.delete(IssuesSchema.getInstance().getTableInfoRelatedIssues(), deleteFilter); + LOG.info(" Deleted {} from RelatedIssues (IssueId)", StringUtilsLabKey.pluralize(deletedRowCount, "row")); + deletedRowCount = Table.delete(IssuesSchema.getInstance().getTableInfoComments(), deleteFilter); + LOG.info(" Deleted {} from Comments", StringUtilsLabKey.pluralize(deletedRowCount, "row")); + deletedRowCount = Table.delete(IssuesSchema.getInstance().getTableInfoIssues(), deleteFilter); + LOG.info(" Deleted {} from Issues", StringUtilsLabKey.pluralize(deletedRowCount, "row")); } @Override diff --git a/list/src/org/labkey/list/ListModule.java b/list/src/org/labkey/list/ListModule.java index c98987eafd6..cf1ae836a0b 100644 --- a/list/src/org/labkey/list/ListModule.java +++ b/list/src/org/labkey/list/ListModule.java @@ -19,8 +19,11 @@ import org.jetbrains.annotations.NotNull; import org.labkey.api.admin.FolderSerializationRegistry; import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.attachments.AttachmentType; import org.labkey.api.audit.AuditLogService; import org.labkey.api.data.Container; +import org.labkey.api.data.DatabaseMigrationService; +import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationSchemaHandler; import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.SqlSelector; @@ -160,6 +163,14 @@ public void startupAfterSpringConfig(ModuleContext moduleContext) return metric; }); } + + DatabaseMigrationService.get().registerSchemaHandler(new DefaultMigrationSchemaHandler(ListSchema.getInstance().getSchema()){ + @Override + public @NotNull Collection getAttachmentTypes() + { + return Set.of(ListItemType.get()); + } + }); } @NotNull diff --git a/list/src/org/labkey/list/view/ListItemType.java b/list/src/org/labkey/list/view/ListItemType.java index cd8aa19fa92..22fd77aee6d 100644 --- a/list/src/org/labkey/list/view/ListItemType.java +++ b/list/src/org/labkey/list/view/ListItemType.java @@ -17,6 +17,7 @@ import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.labkey.api.attachments.AttachmentType; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.SQLFragment; @@ -49,7 +50,7 @@ private ListItemType() } @Override - public void addWhereSql(SQLFragment sql, String parentColumn, String documentNameColumn) + public @Nullable SQLFragment getSelectParentEntityIdsSql() { ListService svc = ListService.get(); assert null != svc; @@ -65,9 +66,13 @@ public void addWhereSql(SQLFragment sql, String parentColumn, String documentNam }); }); + SQLFragment sql = new SQLFragment(); + if (selectStatements.isEmpty()) - sql.append("1 = 0"); // No lists with attachment columns + sql.append("SELECT EntityId WHERE 1 = 0"); // No lists with attachment columns else - sql.append(parentColumn).append(" IN (").append(StringUtils.join(selectStatements, "\n UNION")).append(")"); + sql.append(StringUtils.join(selectStatements, "\n UNION")); + + return sql; } } diff --git a/search/src/org/labkey/search/model/LuceneSearchServiceImpl.java b/search/src/org/labkey/search/model/LuceneSearchServiceImpl.java index 2edcc4a57da..428b5ca9a57 100644 --- a/search/src/org/labkey/search/model/LuceneSearchServiceImpl.java +++ b/search/src/org/labkey/search/model/LuceneSearchServiceImpl.java @@ -1607,7 +1607,7 @@ interface FindHandler } @Override - public WebPartView getSearchView(boolean includeSubfolders, int textBoxWidth, boolean includeHelpLink, boolean isWebpart) + public SearchWebPart getSearchView(boolean includeSubfolders, int textBoxWidth, boolean includeHelpLink, boolean isWebpart) { return new SearchWebPart(includeSubfolders, textBoxWidth, includeHelpLink, isWebpart); } diff --git a/specimen/src/org/labkey/specimen/SpecimenServiceImpl.java b/specimen/src/org/labkey/specimen/SpecimenServiceImpl.java index 3ed5279f565..5be823cc36d 100644 --- a/specimen/src/org/labkey/specimen/SpecimenServiceImpl.java +++ b/specimen/src/org/labkey/specimen/SpecimenServiceImpl.java @@ -19,6 +19,7 @@ import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.labkey.api.attachments.AttachmentType; import org.labkey.api.data.Container; import org.labkey.api.data.DbSchema; import org.labkey.api.data.PropertyManager; @@ -52,6 +53,7 @@ import org.labkey.api.view.ActionURL; import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.specimen.importer.SpecimenColumn; +import org.labkey.specimen.model.SpecimenRequestEventType; import org.labkey.specimen.pipeline.SpecimenReloadJob; import org.labkey.specimen.requirements.SpecimenRequestRequirementProvider; @@ -366,6 +368,12 @@ public void registerRequestCustomizer(SpecimenRequestCustomizer customizer) _specimenRequestCustomizer = customizer; } + @Override + public AttachmentType getSpecimenRequestEventType() + { + return SpecimenRequestEventType.get(); + } + @Override public void fireSpecimensChanged(Container c, User user, Logger logger) { diff --git a/specimen/src/org/labkey/specimen/model/SpecimenRequestEventType.java b/specimen/src/org/labkey/specimen/model/SpecimenRequestEventType.java index a7bcd30cf03..b14f4d1d179 100644 --- a/specimen/src/org/labkey/specimen/model/SpecimenRequestEventType.java +++ b/specimen/src/org/labkey/specimen/model/SpecimenRequestEventType.java @@ -16,6 +16,7 @@ package org.labkey.specimen.model; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.labkey.api.attachments.AttachmentType; import org.labkey.api.data.SQLFragment; import org.labkey.api.specimen.SpecimenSchema; @@ -40,8 +41,8 @@ private SpecimenRequestEventType() } @Override - public void addWhereSql(SQLFragment sql, String parentColumn, String documentNameColumn) + public @Nullable SQLFragment getSelectParentEntityIdsSql() { - sql.append(parentColumn).append(" IN (SELECT EntityId FROM ").append(SpecimenSchema.get().getTableInfoSampleRequestEvent(), "sre").append(")"); + return new SQLFragment("SELECT EntityId FROM ").append(SpecimenSchema.get().getTableInfoSampleRequestEvent(), "sre"); } } \ No newline at end of file diff --git a/study/src/org/labkey/study/StudyModule.java b/study/src/org/labkey/study/StudyModule.java index 9147499cbfa..83b3e546591 100644 --- a/study/src/org/labkey/study/StudyModule.java +++ b/study/src/org/labkey/study/StudyModule.java @@ -542,7 +542,14 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM @Override public @NotNull Collection getAttachmentTypes() { - return List.of(ProtocolDocumentType.get()); + SpecimenService ss = SpecimenService.get(); + + return ss != null ? + List.of( + ProtocolDocumentType.get(), + ss.getSpecimenRequestEventType() + ) : + List.of(ProtocolDocumentType.get()); } }); From 68a1c0d4e021365e4ecfc3e47fedabc19dbc50de Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Mon, 10 Nov 2025 15:43:13 -0800 Subject: [PATCH 07/12] afterMigration() and update core.Documents sequence. Handle data class attachments. --- .../org/labkey/api/attachments/AttachmentType.java | 6 ++++-- .../org/labkey/api/data/DatabaseMigrationService.java | 9 +++++++-- .../org/labkey/core/CoreMigrationSchemaHandler.java | 7 +++++++ .../experiment/DataClassMigrationSchemaHandler.java | 10 ++++++++++ .../org/labkey/experiment/api/ExpDataClassType.java | 10 +++++----- list/src/org/labkey/list/view/ListItemType.java | 11 +++-------- 6 files changed, 36 insertions(+), 17 deletions(-) diff --git a/api/src/org/labkey/api/attachments/AttachmentType.java b/api/src/org/labkey/api/attachments/AttachmentType.java index 31510dc4e20..d9f6a2b67de 100644 --- a/api/src/org/labkey/api/attachments/AttachmentType.java +++ b/api/src/org/labkey/api/attachments/AttachmentType.java @@ -21,10 +21,12 @@ /** * Tags {@link Attachment} objects based on their intended use and what they're attached to. Does not - * necessarily indicate that they are a file of a particular type/format. + * indicate that they are a file of a particular type/format. */ public interface AttachmentType { + SQLFragment NO_ENTITY_IDS = new SQLFragment("SELECT NULL AS EntityId WHERE 1 = 0"); + AttachmentType UNKNOWN = new AttachmentType() { @NotNull @@ -63,7 +65,7 @@ default void addWhereSql(SQLFragment sql, String parentColumn, String documentNa * provide attachments of this type, without involving the core.Documents table. For example, * {@code SELECT EntityId FROM comm.Announcements}. Return null if this is not-yet-implemented or inappropriate. * For example, some attachments' parents are container IDs. If the method determines that no parents exist, then - * return a valid query that selects no rows, for example, {@code SELECT EntityID WHERE 1 = 0}. + * return a valid query that selects no rows, for example, {@code NO_ENTITY_IDS}. */ default @Nullable SQLFragment getSelectParentEntityIdsSql() { diff --git a/api/src/org/labkey/api/data/DatabaseMigrationService.java b/api/src/org/labkey/api/data/DatabaseMigrationService.java index 66754be885f..6cb8be5e5f6 100644 --- a/api/src/org/labkey/api/data/DatabaseMigrationService.java +++ b/api/src/org/labkey/api/data/DatabaseMigrationService.java @@ -111,6 +111,8 @@ interface MigrationSchemaHandler void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema); @NotNull Collection getAttachmentTypes(); + + void afterMigration(DatabaseMigrationConfiguration configuration); } class DefaultMigrationSchemaHandler implements MigrationSchemaHandler @@ -327,9 +329,7 @@ public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSche copyAttachments(configuration, sourceSchema, new SQLClause(selectParents), type); } - // TODO: **Test issues attachment copy // TODO: implement remaining AttachmentTypes - // TODO: afterMigration() and update core.Documents' sequence // TODO: throw if some registered AttachmentType is not seen // TODO: fail if type.getSelectParentEntityIdsSql() returns null }); @@ -378,6 +378,11 @@ public static void logUnseenAttachmentTypes() public void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema) { } + + @Override + public void afterMigration(DatabaseMigrationConfiguration configuration) + { + } } /** diff --git a/core/src/org/labkey/core/CoreMigrationSchemaHandler.java b/core/src/org/labkey/core/CoreMigrationSchemaHandler.java index b2248744dac..c137724b2a8 100644 --- a/core/src/org/labkey/core/CoreMigrationSchemaHandler.java +++ b/core/src/org/labkey/core/CoreMigrationSchemaHandler.java @@ -211,6 +211,13 @@ public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSche ); } + @Override + public void afterMigration(DatabaseMigrationConfiguration configuration) + { + // Now that all schemas have copied their attachments into core.Documents, update that table's sequence + DatabaseMigrationService.get().updateSequences(configuration.getSourceScope().getSchema("core", DbSchemaType.Migration).getTable("Documents"), CoreSchema.getInstance().getTableInfoDocuments()); + } + // MigrationFilter implementation below private SQLFragment _groupFilterCondition = null; diff --git a/experiment/src/org/labkey/experiment/DataClassMigrationSchemaHandler.java b/experiment/src/org/labkey/experiment/DataClassMigrationSchemaHandler.java index 140d9ccfab6..79793151405 100644 --- a/experiment/src/org/labkey/experiment/DataClassMigrationSchemaHandler.java +++ b/experiment/src/org/labkey/experiment/DataClassMigrationSchemaHandler.java @@ -1,6 +1,8 @@ package org.labkey.experiment; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.attachments.AttachmentType; import org.labkey.api.collections.Sets; import org.labkey.api.data.DatabaseMigrationConfiguration; import org.labkey.api.data.DatabaseMigrationService; @@ -27,10 +29,12 @@ import org.labkey.api.util.StringUtilsLabKey; import org.labkey.api.util.logging.LogHelper; import org.labkey.experiment.api.DataClassDomainKind; +import org.labkey.experiment.api.ExpDataClassType; import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Set; class DataClassMigrationSchemaHandler extends DefaultMigrationSchemaHandler implements ExperimentDeleteService @@ -188,4 +192,10 @@ public FilterClause getTableFilterClause(TableInfo sourceTable, Set contai }); } } + + @Override + public @NotNull Collection getAttachmentTypes() + { + return List.of(ExpDataClassType.get()); + } } diff --git a/experiment/src/org/labkey/experiment/api/ExpDataClassType.java b/experiment/src/org/labkey/experiment/api/ExpDataClassType.java index 5bf36aeea71..be783a6b625 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataClassType.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataClassType.java @@ -17,6 +17,7 @@ import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.labkey.api.attachments.AttachmentType; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; @@ -54,7 +55,7 @@ public static AttachmentType get() } @Override - public void addWhereSql(SQLFragment sql, String parentColumn, String documentNameColumn) + public @Nullable SQLFragment getSelectParentEntityIdsSql() { TableInfo tableInfo = ExperimentService.get().getTinfoDataClass(); @@ -79,10 +80,9 @@ public void addWhereSql(SQLFragment sql, String parentColumn, String documentNam selectStatements.add("\n SELECT " + expressionToExtractObjectId + " AS ID FROM expdataclass." + domain.getStorageTableName() + " WHERE " + where); }); - if (selectStatements.isEmpty()) - sql.append("1 = 0"); // No ExpDataClasses with attachment columns - else - sql.append(parentColumn).append(" IN (").append(StringUtils.join(selectStatements, "\n UNION")).append(")"); + return selectStatements.isEmpty() ? + NO_ENTITY_IDS : // No ExpDataClasses with attachment columns + new SQLFragment(StringUtils.join(selectStatements, "\n UNION")); } } diff --git a/list/src/org/labkey/list/view/ListItemType.java b/list/src/org/labkey/list/view/ListItemType.java index 22fd77aee6d..c1e0b596304 100644 --- a/list/src/org/labkey/list/view/ListItemType.java +++ b/list/src/org/labkey/list/view/ListItemType.java @@ -66,13 +66,8 @@ private ListItemType() }); }); - SQLFragment sql = new SQLFragment(); - - if (selectStatements.isEmpty()) - sql.append("SELECT EntityId WHERE 1 = 0"); // No lists with attachment columns - else - sql.append(StringUtils.join(selectStatements, "\n UNION")); - - return sql; + return selectStatements.isEmpty() ? + NO_ENTITY_IDS : // No lists with attachment columns + new SQLFragment(StringUtils.join(selectStatements, "\n UNION")); } } From 501f48e3d04d8505bb0af60bdec64b13a3176e7b Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Mon, 10 Nov 2025 16:03:25 -0800 Subject: [PATCH 08/12] Handle expProtocol and expRun attachments --- .../api/exp/api/ExpProtocolAttachmentParent.java | 2 +- .../api/exp/api/ExpProtocolAttachmentType.java | 5 +++-- .../labkey/api/exp/api/ExpRunAttachmentType.java | 5 +++-- .../ExperimentMigrationSchemaHandler.java | 14 ++++++++++++++ study/src/org/labkey/study/StudyModule.java | 11 +++++++++++ 5 files changed, 32 insertions(+), 5 deletions(-) diff --git a/api/src/org/labkey/api/exp/api/ExpProtocolAttachmentParent.java b/api/src/org/labkey/api/exp/api/ExpProtocolAttachmentParent.java index d1340315f73..fef7e961471 100644 --- a/api/src/org/labkey/api/exp/api/ExpProtocolAttachmentParent.java +++ b/api/src/org/labkey/api/exp/api/ExpProtocolAttachmentParent.java @@ -43,6 +43,6 @@ public String getContainerId() @Override public @NotNull AttachmentType getAttachmentType() { - return ExpRunAttachmentType.get(); + return ExpProtocolAttachmentType.get(); } } diff --git a/api/src/org/labkey/api/exp/api/ExpProtocolAttachmentType.java b/api/src/org/labkey/api/exp/api/ExpProtocolAttachmentType.java index 25ee35b78f3..65cd70f0969 100644 --- a/api/src/org/labkey/api/exp/api/ExpProtocolAttachmentType.java +++ b/api/src/org/labkey/api/exp/api/ExpProtocolAttachmentType.java @@ -16,6 +16,7 @@ package org.labkey.api.exp.api; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.labkey.api.attachments.AttachmentType; import org.labkey.api.data.SQLFragment; @@ -39,8 +40,8 @@ private ExpProtocolAttachmentType() } @Override - public void addWhereSql(SQLFragment sql, String parentColumn, String documentNameColumn) + public @Nullable SQLFragment getSelectParentEntityIdsSql() { - sql.append(parentColumn).append(" IN (SELECT EntityId FROM ").append(ExperimentService.get().getTinfoProtocol(), "ep").append(")"); + return new SQLFragment("SELECT EntityId FROM ").append(ExperimentService.get().getTinfoProtocol(), "ep"); } } \ No newline at end of file diff --git a/api/src/org/labkey/api/exp/api/ExpRunAttachmentType.java b/api/src/org/labkey/api/exp/api/ExpRunAttachmentType.java index 9be08401bfc..3e9fb3b298c 100644 --- a/api/src/org/labkey/api/exp/api/ExpRunAttachmentType.java +++ b/api/src/org/labkey/api/exp/api/ExpRunAttachmentType.java @@ -16,6 +16,7 @@ package org.labkey.api.exp.api; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.labkey.api.attachments.AttachmentType; import org.labkey.api.data.SQLFragment; @@ -39,8 +40,8 @@ private ExpRunAttachmentType() } @Override - public void addWhereSql(SQLFragment sql, String parentColumn, String documentNameColumn) + public @Nullable SQLFragment getSelectParentEntityIdsSql() { - sql.append(parentColumn).append(" IN (SELECT EntityId FROM ").append(ExperimentService.get().getTinfoExperimentRun(), "er").append(")"); + return new SQLFragment("SELECT EntityId FROM ").append(ExperimentService.get().getTinfoExperimentRun(), "er"); } } \ No newline at end of file diff --git a/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java b/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java index 69399581f3f..ce5285694ef 100644 --- a/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java +++ b/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java @@ -1,7 +1,9 @@ package org.labkey.experiment; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.labkey.api.attachments.AttachmentType; import org.labkey.api.data.CompareType; import org.labkey.api.data.CompareType.CompareClause; import org.labkey.api.data.DatabaseMigrationConfiguration; @@ -16,11 +18,14 @@ import org.labkey.api.data.SqlExecutor; import org.labkey.api.data.TableInfo; import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.api.ExpProtocolAttachmentType; +import org.labkey.api.exp.api.ExpRunAttachmentType; import org.labkey.api.query.FieldKey; import org.labkey.api.util.GUID; import org.labkey.api.util.logging.LogHelper; import org.labkey.experiment.api.ExperimentServiceImpl; +import java.util.Collection; import java.util.List; import java.util.Set; @@ -154,4 +159,13 @@ public static void deleteObjectIds(SQLFragment objectIdClause) .append(objectIdClause) ); } + + @Override + public @NotNull Collection getAttachmentTypes() + { + return List.of( + ExpProtocolAttachmentType.get(), + ExpRunAttachmentType.get() + ); + } } diff --git a/study/src/org/labkey/study/StudyModule.java b/study/src/org/labkey/study/StudyModule.java index 83b3e546591..0c1a13eac8a 100644 --- a/study/src/org/labkey/study/StudyModule.java +++ b/study/src/org/labkey/study/StudyModule.java @@ -76,6 +76,7 @@ import org.labkey.api.settings.OptionalFeatureService; import org.labkey.api.specimen.SpecimenManager; import org.labkey.api.specimen.SpecimenSampleTypeDomainKind; +import org.labkey.api.specimen.SpecimenSchema; import org.labkey.api.specimen.model.AdditiveTypeDomainKind; import org.labkey.api.specimen.model.DerivativeTypeDomainKind; import org.labkey.api.specimen.model.LocationDomainKind; @@ -562,6 +563,16 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM return SITE_WIDE_TABLE; } }); + + DatabaseMigrationService.get().registerSchemaHandler(new DefaultMigrationSchemaHandler(SpecimenSchema.get().getSchema()) + { + @Override + public @Nullable FieldKey getContainerFieldKey(TableInfo sourceTable) + { + // The specimen tables lack both a container column and an FK to a table that does, but they're single-container tables + return sourceTable.getName().endsWith("_specimen") ? SITE_WIDE_TABLE : super.getContainerFieldKey(sourceTable); + } + }); } @Override From 32015f4d31b07f44ab397dca6c8adfa3d23986f7 Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Mon, 10 Nov 2025 16:14:45 -0800 Subject: [PATCH 09/12] Specimen --- study/src/org/labkey/study/StudyModule.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/study/src/org/labkey/study/StudyModule.java b/study/src/org/labkey/study/StudyModule.java index 0c1a13eac8a..2d5358dd782 100644 --- a/study/src/org/labkey/study/StudyModule.java +++ b/study/src/org/labkey/study/StudyModule.java @@ -31,6 +31,8 @@ import org.labkey.api.data.ContainerManager; import org.labkey.api.data.DatabaseMigrationService; import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationSchemaHandler; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.PropertySchema; import org.labkey.api.data.SqlExecutor; import org.labkey.api.data.SqlSelector; @@ -76,13 +78,13 @@ import org.labkey.api.settings.OptionalFeatureService; import org.labkey.api.specimen.SpecimenManager; import org.labkey.api.specimen.SpecimenSampleTypeDomainKind; -import org.labkey.api.specimen.SpecimenSchema; import org.labkey.api.specimen.model.AdditiveTypeDomainKind; import org.labkey.api.specimen.model.DerivativeTypeDomainKind; import org.labkey.api.specimen.model.LocationDomainKind; import org.labkey.api.specimen.model.PrimaryTypeDomainKind; import org.labkey.api.specimen.model.SpecimenDomainKind; import org.labkey.api.specimen.model.SpecimenEventDomainKind; +import org.labkey.api.specimen.model.SpecimenTablesProvider; import org.labkey.api.specimen.model.VialDomainKind; import org.labkey.api.study.ParticipantCategory; import org.labkey.api.study.SpecimenService; @@ -564,12 +566,12 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM } }); - DatabaseMigrationService.get().registerSchemaHandler(new DefaultMigrationSchemaHandler(SpecimenSchema.get().getSchema()) + DatabaseMigrationService.get().registerSchemaHandler(new DefaultMigrationSchemaHandler(DbSchema.get(SpecimenTablesProvider.SCHEMA_NAME, DbSchemaType.Provisioned)) { @Override public @Nullable FieldKey getContainerFieldKey(TableInfo sourceTable) { - // The specimen tables lack both a container column and an FK to a table that does, but they're single-container tables + // The "_specimen" tables lack both a container column and an FK to a table that does, but they're single-container tables return sourceTable.getName().endsWith("_specimen") ? SITE_WIDE_TABLE : super.getContainerFieldKey(sourceTable); } }); From 8fa79ffd4464400c5b41843c4afdc96540f579d8 Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Mon, 10 Nov 2025 20:20:25 -0800 Subject: [PATCH 10/12] Repackage migration interfaces --- .../announcements/AnnouncementModule.java | 6 +- .../api/data/DatabaseMigrationService.java | 442 ------------------ api/src/org/labkey/api/data/DbSchemaType.java | 1 + .../api/files}/FileSystemAttachmentType.java | 95 ++-- .../labkey/api/migration/AssaySkipFilter.java | 33 ++ .../DatabaseMigrationConfiguration.java | 7 +- .../migration/DatabaseMigrationService.java | 83 ++++ .../DefaultMigrationSchemaHandler.java | 306 ++++++++++++ .../migration/ExperimentDeleteService.java | 27 ++ .../MigrationDbSchema.java | 6 +- .../labkey/api/migration/MigrationFilter.java | 17 + .../api/migration/MigrationSchemaHandler.java | 58 +++ .../org/labkey/api/module/ModuleLoader.java | 4 +- assay/src/org/labkey/assay/AssayModule.java | 4 +- .../AssayResultMigrationSchemaHandler.java | 55 +-- .../core/CoreMigrationSchemaHandler.java | 11 +- .../DataClassMigrationSchemaHandler.java | 10 +- .../ExperimentMigrationSchemaHandler.java | 4 +- .../labkey/experiment/ExperimentModule.java | 7 +- .../SampleTypeMigrationSchemaHandler.java | 4 +- .../labkey/filecontent/FileContentModule.java | 1 + .../FileSystemAttachmentParent.java | 2 +- .../issue/IssueMigrationSchemaHandler.java | 4 +- issues/src/org/labkey/issue/IssuesModule.java | 2 +- list/src/org/labkey/list/ListModule.java | 4 +- .../src/org/labkey/search/SearchModule.java | 4 +- study/src/org/labkey/study/StudyModule.java | 4 +- 27 files changed, 651 insertions(+), 550 deletions(-) delete mode 100644 api/src/org/labkey/api/data/DatabaseMigrationService.java rename {filecontent/src/org/labkey/filecontent => api/src/org/labkey/api/files}/FileSystemAttachmentType.java (79%) create mode 100644 api/src/org/labkey/api/migration/AssaySkipFilter.java rename api/src/org/labkey/api/{data => migration}/DatabaseMigrationConfiguration.java (89%) create mode 100644 api/src/org/labkey/api/migration/DatabaseMigrationService.java create mode 100644 api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java create mode 100644 api/src/org/labkey/api/migration/ExperimentDeleteService.java rename api/src/org/labkey/api/{data => migration}/MigrationDbSchema.java (76%) create mode 100644 api/src/org/labkey/api/migration/MigrationFilter.java create mode 100644 api/src/org/labkey/api/migration/MigrationSchemaHandler.java diff --git a/announcements/src/org/labkey/announcements/AnnouncementModule.java b/announcements/src/org/labkey/announcements/AnnouncementModule.java index 6125aa11f12..9c0d618d1d6 100644 --- a/announcements/src/org/labkey/announcements/AnnouncementModule.java +++ b/announcements/src/org/labkey/announcements/AnnouncementModule.java @@ -36,14 +36,14 @@ import org.labkey.api.audit.provider.MessageAuditProvider; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DatabaseMigrationConfiguration; -import org.labkey.api.data.DatabaseMigrationService; -import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationSchemaHandler; import org.labkey.api.data.DbSchema; import org.labkey.api.data.SqlExecutor; import org.labkey.api.data.TableInfo; import org.labkey.api.message.digest.DailyMessageDigest; import org.labkey.api.message.settings.MessageConfigService; +import org.labkey.api.migration.DatabaseMigrationConfiguration; +import org.labkey.api.migration.DatabaseMigrationService; +import org.labkey.api.migration.DefaultMigrationSchemaHandler; import org.labkey.api.module.DefaultModule; import org.labkey.api.module.ModuleContext; import org.labkey.api.rss.RSSService; diff --git a/api/src/org/labkey/api/data/DatabaseMigrationService.java b/api/src/org/labkey/api/data/DatabaseMigrationService.java deleted file mode 100644 index 6cb8be5e5f6..00000000000 --- a/api/src/org/labkey/api/data/DatabaseMigrationService.java +++ /dev/null @@ -1,442 +0,0 @@ -package org.labkey.api.data; - -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.attachments.AttachmentType; -import org.labkey.api.data.DatabaseMigrationConfiguration.DefaultDatabaseMigrationConfiguration; -import org.labkey.api.data.SimpleFilter.AndClause; -import org.labkey.api.data.SimpleFilter.FilterClause; -import org.labkey.api.data.SimpleFilter.InClause; -import org.labkey.api.data.SimpleFilter.OrClause; -import org.labkey.api.data.SimpleFilter.SQLClause; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.TableSorter; -import org.labkey.api.services.ServiceRegistry; -import org.labkey.api.util.ConfigurationException; -import org.labkey.api.util.GUID; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.vfs.FileLike; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -public interface DatabaseMigrationService -{ - Logger LOG = LogHelper.getLogger(DatabaseMigrationService.class, "Information about database migration"); - - record DataFilter(Set containers, String column, FilterClause condition) {} - - static @NotNull DatabaseMigrationService get() - { - DatabaseMigrationService ret = ServiceRegistry.get().getService(DatabaseMigrationService.class); - return ret != null ? ret : new DatabaseMigrationService() {}; - } - - static void setInstance(DatabaseMigrationService impl) - { - ServiceRegistry.get().registerService(DatabaseMigrationService.class, impl); - } - - default DatabaseMigrationConfiguration getDatabaseMigrationConfiguration(FileLike labkeyRoot, @Nullable String migration) - { - return new DefaultDatabaseMigrationConfiguration(); - } - - // By default, no-op implementation that simply logs - default void migrate(DatabaseMigrationConfiguration configuration) - { - LOG.warn("Database migration service is not present; database migration is a premium feature."); - } - - // By default, no-op implementations - default void registerSchemaHandler(MigrationSchemaHandler schemaHandler) {} - default void registerMigrationFilter(MigrationFilter filter) {} - - default @Nullable MigrationFilter getMigrationFilter(String propertyName) - { - return null; - } - - default void copySourceTableToTargetTable(DatabaseMigrationConfiguration configuration, TableInfo sourceTable, TableInfo targetTable, DbSchemaType schemaType, boolean updateSequences, String additionalLogMessage, MigrationSchemaHandler schemaHandler) {} - default void updateSequences(TableInfo sourceTable, TableInfo targetTable) {} - - interface MigrationSchemaHandler - { - // Marker for tables to declare themselves as site-wide (no container filtering) - FieldKey SITE_WIDE_TABLE = FieldKey.fromParts("site-wide"); - - DbSchema getSchema(); - - void beforeVerification(); - - void beforeSchema(); - - List getTablesToCopy(); - - // Create a filter clause that selects from all specified containers and (in some overrides) applies table-specific filters - FilterClause getTableFilterClause(TableInfo sourceTable, Set containers); - - // Create a filter clause that selects from all specified containers - FilterClause getContainerClause(TableInfo sourceTable, Set containers); - - // Return the FieldKey that can be used to filter this table by container. Special values SITE_WIDE_TABLE and - // DUMMY_FIELD_KEY can be returned for special behaviors. DUMMY_FIELD_KEY ensures that the handler's custom - // getContainerClause() is always called. SITE_WIDE_TABLE is used to select all rows. - @Nullable FieldKey getContainerFieldKey(TableInfo sourceTable); - - // Create a filter clause that selects all rows from unfiltered containers plus filtered rows from the filtered containers - FilterClause getDomainDataFilterClause(Set copyContainers, Set filteredContainers, List domainFilters, TableInfo sourceTable, Set selectColumnNames); - - void addDomainDataFilterClause(OrClause orClause, DataFilter filter, TableInfo sourceTable, Set selectColumnNames); - - // Do any necessary clean up after the target table has been populated. notCopiedFilter selects all rows in the - // source table that were NOT copied to the target table. (For example, rows in a global table not copied due to - // container filtering or rows in a provisioned table not copied due to domain data filtering.) - void afterTable(TableInfo sourceTable, TableInfo targetTable, SimpleFilter notCopiedFilter); - - void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema); - - // TODO: Return Collection, indicating which attachment types it handled? - void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema); - - @NotNull Collection getAttachmentTypes(); - - void afterMigration(DatabaseMigrationConfiguration configuration); - } - - class DefaultMigrationSchemaHandler implements MigrationSchemaHandler - { - private final DbSchema _schema; - - public DefaultMigrationSchemaHandler(DbSchema schema) - { - _schema = schema; - } - - @Override - public DbSchema getSchema() - { - return _schema; - } - - @Override - public void beforeVerification() - { - } - - @Override - public void beforeSchema() - { - } - - @Override - public List getTablesToCopy() - { - Set sortedTables = new LinkedHashSet<>(TableSorter.sort(getSchema(), true)); - - Set allTables = getSchema().getTableNames().stream() - .map(getSchema()::getTable) - .collect(Collectors.toCollection(HashSet::new)); - allTables.removeAll(sortedTables); - - if (!allTables.isEmpty()) - { - LOG.info("These tables were removed by TableSorter: {}", allTables); - } - - return sortedTables.stream() - // Skip all views and virtual tables (e.g., test.Containers2, which is a table on SS but a view on PG) - .filter(table -> table.getTableType() == DatabaseTableType.TABLE) - .collect(Collectors.toCollection(ArrayList::new)); // Ensure mutable - } - - @Override - public FilterClause getTableFilterClause(TableInfo sourceTable, Set containers) - { - return getContainerClause(sourceTable, containers); - } - - @Override - public FilterClause getContainerClause(TableInfo sourceTable, Set containers) - { - FieldKey containerFieldKey = getContainerFieldKey(sourceTable); - - if (containerFieldKey == SITE_WIDE_TABLE) - return new SQLClause(new SQLFragment("TRUE")); - - return new InClause(containerFieldKey, containers); - } - - @Override - public @Nullable FieldKey getContainerFieldKey(TableInfo table) - { - FieldKey fKey = table.getContainerFieldKey(); - - if (fKey != null) - return fKey; - - for (ColumnInfo col : table.getColumns()) - { - ForeignKey fk = TableSorter.getForeignKey(table, col, true); - if (fk != null) - { - // Use the table's schema (or a migration schema retrieved from the table's scope), since we want a Migration schema with XML metadata applied - DbSchema tableSchema = table.getSchema(); - DbSchema lookupSchema = fk.getLookupSchemaKey().equals(new SchemaKey(null, tableSchema.getName())) ? - tableSchema : - tableSchema.getScope().getSchema(fk.getLookupSchemaName(), DbSchemaType.Migration); - TableInfo lookupTableInfo = lookupSchema.getTable(fk.getLookupTableName()); - if (lookupTableInfo != null) - { - fKey = lookupTableInfo.getContainerFieldKey(); - - if (null == fKey) - { - // Ignore self joins - if (!lookupTableInfo.getName().equalsIgnoreCase(table.getName())) - { - fKey = getContainerFieldKey(lookupTableInfo); - } - } - - if (fKey != null) - return FieldKey.fromParts(col.getFieldKey(), fKey); - } - } - } - - return null; - } - - @Override - public final FilterClause getDomainDataFilterClause(Set copyContainers, Set filteredContainers, List domainFilters, TableInfo sourceTable, Set selectColumnNames) - { - // Filtered case: remove the filtered containers from the unconditional container set - Set otherContainers = new HashSet<>(copyContainers); - otherContainers.removeAll(filteredContainers); - FilterClause ret = getContainerClause(sourceTable, otherContainers); - - OrClause orClause = new OrClause(); - - // Delegate to the MigrationSchemaHandler to add domain-filtered containers back with their special filter applied - domainFilters.forEach(filter -> addDomainDataFilterClause(orClause, filter, sourceTable, selectColumnNames)); - - if (!orClause.getClauses().isEmpty()) - { - orClause.addClause(ret); - ret = orClause; - } - - return ret; - } - - @Override - public void addDomainDataFilterClause(OrClause orClause, DataFilter filter, TableInfo sourceTable, Set selectColumnNames) - { - addDataFilterClause(orClause, filter, sourceTable, selectColumnNames); - } - - // Add a filter and return true if the column exists directly on the table - protected boolean addDataFilterClause(OrClause orClause, DataFilter filter, TableInfo sourceTable, Set selectColumnNames) - { - boolean columnExists = selectColumnNames.contains(filter.column()); - - if (columnExists) - { - // Select all rows in this domain-filtered container that meet its criteria - orClause.addClause( - new AndClause( - getContainerClause(sourceTable, filter.containers()), - filter.condition() - ) - ); - } - - return columnExists; - } - - // Add a clause that selects all rows where the object property with equals the filter value. This - // is only for provisioned tables that lack an ObjectId, MaterialId, or DataId column. - protected void addObjectPropertyClause(OrClause orClause, DataFilter filter, TableInfo sourceTable, int propertyId) - { - SQLFragment flagWhere = new SQLFragment("lsid IN (SELECT ObjectURI FROM exp.Object o INNER JOIN exp.ObjectProperty op ON o.ObjectId = op.ObjectId WHERE StringValue = ? AND PropertyId = ?)", filter.condition().getParamVals()[0], propertyId); - - orClause.addClause( - new AndClause( - getContainerClause(sourceTable, filter.containers()), - new SQLClause(flagWhere) - ) - ); - } - - private Integer _commentPropertyId = null; - - protected synchronized int getCommentPropertyId(DbScope scope) - { - if (_commentPropertyId == null) - { - // Get the exp.PropertyDescriptor table from the source scope - TableInfo propertyDescriptor = scope.getSchema("exp", DbSchemaType.Migration).getTable("PropertyDescriptor"); - // Select the PropertyId associated with built-in Flag fields ("urn:exp.labkey.org/#Comment") - Integer propertyId = new TableSelector(propertyDescriptor, Collections.singleton("PropertyId"), new SimpleFilter(FieldKey.fromParts("PropertyURI"), "urn:exp.labkey.org/#Comment"), null).getObject(Integer.class); - if (propertyId == null) - throw new RuntimeException("PropertyDescriptor for built-in Flag field not found"); - else - _commentPropertyId = propertyId; - } - - return _commentPropertyId; - } - - protected String rowsNotCopied(int count) - { - return " " + StringUtilsLabKey.pluralize(count, "row") + " not copied"; - } - - @Override - public void afterTable(TableInfo sourceTable, TableInfo targetTable, SimpleFilter notCopiedFilter) - { - } - - @Override - public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema) - { - // Now that the target tables in this schema have been populated, copy all associated attachments. By - // default, use this handler's attachment types to select from the target tables all EntityIds that might be - // attachment parents (this avoids re-running potentially expensive queries on the source tables). Use the - // set of EntityIds to copy those attachments from the core.Documents table in the source database. Override - // if special behavior is required, for example, AttachmentTypes that use documentNameColumn since that - // requires querying and re-filtering the source tables instead. - getAttachmentTypes().forEach(type -> { - SQLFragment sql = type.getSelectParentEntityIdsSql(); - if (sql != null) - { - Collection entityIds = new SqlSelector(targetSchema, sql).getCollection(String.class); - SQLFragment selectParents = new SQLFragment("Parent"); - // This query against the source database is likely to contain a large IN clause, so use an alternative InClauseGenerator - sourceSchema.getSqlDialect().appendInClauseSql(selectParents, entityIds, getTempTableInClauseGenerator(sourceSchema.getScope())); - copyAttachments(configuration, sourceSchema, new SQLClause(selectParents), type); - } - - // TODO: implement remaining AttachmentTypes - // TODO: throw if some registered AttachmentType is not seen - // TODO: fail if type.getSelectParentEntityIdsSql() returns null - }); - } - - // Creates a TempTableInClauseGenerator that targets the *source* temp schema instead of the default - // DbSchema.getTemp(). Required for large IN clauses used against the source database. - protected InClauseGenerator getTempTableInClauseGenerator(DbScope sourceScope) - { - return new TempTableInClauseGenerator(() -> sourceScope.getSchema("temp", DbSchemaType.Bare)); - } - - private static final Set SEEN = new HashSet<>(); - - // Copy all core.Documents rows that match the provided filter clause - protected void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, FilterClause filterClause, AttachmentType... type) - { - SEEN.addAll(Arrays.asList(type)); - String additionalMessage = " associated with " + Arrays.stream(type).map(t -> t.getClass().getSimpleName()).collect(Collectors.joining(", ")); - TableInfo sourceDocumentsTable = sourceSchema.getScope().getSchema("core", DbSchemaType.Migration).getTable("Documents"); - TableInfo targetDocumentsTable = CoreSchema.getInstance().getTableInfoDocuments(); - DatabaseMigrationService.get().copySourceTableToTargetTable(configuration, sourceDocumentsTable, targetDocumentsTable, DbSchemaType.Module, false, additionalMessage, new DefaultMigrationSchemaHandler(CoreSchema.getInstance().getSchema()){ - @Override - public FilterClause getTableFilterClause(TableInfo sourceTable, Set containers) - { - return filterClause; - } - }); - } - - public static void logUnseenAttachmentTypes() - { - Set unseen = new HashSet<>(AttachmentService.get().getAttachmentTypes()); - unseen.removeAll(SEEN); - - LOG.info("These AttachmentTypes have not been seen: {}", unseen.stream().map(type -> type.getClass().getSimpleName()).collect(Collectors.joining(", "))); - } - - @Override - public @NotNull Collection getAttachmentTypes() - { - return List.of(); - } - - @Override - public void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema) - { - } - - @Override - public void afterMigration(DatabaseMigrationConfiguration configuration) - { - } - } - - /** - * A MigrationFilter adds support for the named filter property in the migration configuration file. If present, - * saveFilter() is called with the container guid and property value. Modules can register these to present - * module-specific filters. - */ - interface MigrationFilter - { - String getName(); - // Implementations should validate guid nullity - void saveFilter(@Nullable GUID guid, String value); - } - - interface ExperimentDeleteService - { - static @NotNull ExperimentDeleteService get() - { - ExperimentDeleteService ret = ServiceRegistry.get().getService(ExperimentDeleteService.class); - if (ret == null) - throw new IllegalStateException("ExperimentDeleteService not found"); - return ret; - } - - static void setInstance(ExperimentDeleteService impl) - { - ServiceRegistry.get().registerService(ExperimentDeleteService.class, impl); - } - - /** - * Deletes all rows from exp.Data, exp.Object, and related tables associated with the provided ObjectIds - */ - void deleteDataRows(Collection objectIds); - } - - // Helper method that parses a data filter then adds it and its container to the provided collections, coalescing - // cases where multiple containers specify the same filter - static void addDataFilter(String filterName, List dataFilters, Set filteredContainers, GUID guid, String filter) - { - String[] filterParts = filter.split("="); - if (filterParts.length != 2) - throw new ConfigurationException("Bad " + filterName + " value; expected =: " + filter); - - if (!filteredContainers.add(guid)) - throw new ConfigurationException("Duplicate " + filterName + " entry for container " + guid); - - String column = filterParts[0]; - String value = filterParts[1]; - FilterClause clause = CompareType.EQUAL.createFilterClause(new FieldKey(null, column), value); - // If another container is already using this filter clause, then simply add this guid to that domain filter. - // Otherwise, add a new domain filter to the list. - dataFilters.stream() - .filter(df -> df.column().equals(column) && df.condition().equals(clause)) - .findFirst() - .ifPresentOrElse(df -> df.containers().add(guid), () -> dataFilters.add(new DataFilter(new HashSet<>(Set.of(guid)), filterParts[0], clause))); - } -} diff --git a/api/src/org/labkey/api/data/DbSchemaType.java b/api/src/org/labkey/api/data/DbSchemaType.java index 208dc2a61b9..cd54c930c5d 100644 --- a/api/src/org/labkey/api/data/DbSchemaType.java +++ b/api/src/org/labkey/api/data/DbSchemaType.java @@ -19,6 +19,7 @@ import org.jetbrains.annotations.Nullable; import org.labkey.api.cache.CacheManager; import org.labkey.api.exp.api.ProvisionedDbSchema; +import org.labkey.api.migration.MigrationDbSchema; import org.labkey.api.module.Module; import org.labkey.api.module.ModuleLoader; diff --git a/filecontent/src/org/labkey/filecontent/FileSystemAttachmentType.java b/api/src/org/labkey/api/files/FileSystemAttachmentType.java similarity index 79% rename from filecontent/src/org/labkey/filecontent/FileSystemAttachmentType.java rename to api/src/org/labkey/api/files/FileSystemAttachmentType.java index ec7ac0a22e5..b39e09dba99 100644 --- a/filecontent/src/org/labkey/filecontent/FileSystemAttachmentType.java +++ b/api/src/org/labkey/api/files/FileSystemAttachmentType.java @@ -1,47 +1,48 @@ -/* - * Copyright (c) 2017 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.filecontent; - -import org.jetbrains.annotations.NotNull; -import org.labkey.api.attachments.AttachmentType; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.SQLFragment; - -public class FileSystemAttachmentType implements AttachmentType -{ - private static final FileSystemAttachmentType INSTANCE = new FileSystemAttachmentType(); - - public static FileSystemAttachmentType get() - { - return INSTANCE; - } - - private FileSystemAttachmentType() - { - } - - @Override - public @NotNull String getUniqueName() - { - return getClass().getName(); - } - - @Override - public void addWhereSql(SQLFragment sql, String parentColumn, String documentNameColumn) - { - sql.append(parentColumn).append(" IN (SELECT EntityId FROM ").append(CoreSchema.getInstance().getMappedDirectories(), "md").append(")"); - } -} +/* + * Copyright (c) 2017 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.files; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.attachments.AttachmentType; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.SQLFragment; + +public class FileSystemAttachmentType implements AttachmentType +{ + private static final FileSystemAttachmentType INSTANCE = new FileSystemAttachmentType(); + + public static FileSystemAttachmentType get() + { + return INSTANCE; + } + + private FileSystemAttachmentType() + { + } + + @Override + public @NotNull String getUniqueName() + { + return getClass().getName(); + } + + @Override + public @Nullable SQLFragment getSelectParentEntityIdsSql() + { + return new SQLFragment("SELECT EntityId FROM ").append(CoreSchema.getInstance().getMappedDirectories(), "md"); + } +} diff --git a/api/src/org/labkey/api/migration/AssaySkipFilter.java b/api/src/org/labkey/api/migration/AssaySkipFilter.java new file mode 100644 index 00000000000..1f0b7c1e3dd --- /dev/null +++ b/api/src/org/labkey/api/migration/AssaySkipFilter.java @@ -0,0 +1,33 @@ +package org.labkey.api.migration; + +import org.jetbrains.annotations.Nullable; +import org.labkey.api.util.ConfigurationException; +import org.labkey.api.util.GUID; + +import java.util.HashSet; +import java.util.Set; + +public class AssaySkipFilter implements MigrationFilter +{ + private static final Set SKIP_CONTAINERS = new HashSet<>(); + + @Override + public String getName() + { + return "AssaySkipFilter"; + } + + @Override + public void saveFilter(@Nullable GUID guid, String value) + { + if (null == guid) + throw new ConfigurationException(getName() + " must specify a GUID"); + + SKIP_CONTAINERS.add(guid); + } + + public static Set getSkipContainers() + { + return SKIP_CONTAINERS; + } +} diff --git a/api/src/org/labkey/api/data/DatabaseMigrationConfiguration.java b/api/src/org/labkey/api/migration/DatabaseMigrationConfiguration.java similarity index 89% rename from api/src/org/labkey/api/data/DatabaseMigrationConfiguration.java rename to api/src/org/labkey/api/migration/DatabaseMigrationConfiguration.java index 26125f4c287..1d92782419b 100644 --- a/api/src/org/labkey/api/data/DatabaseMigrationConfiguration.java +++ b/api/src/org/labkey/api/migration/DatabaseMigrationConfiguration.java @@ -1,8 +1,11 @@ -package org.labkey.api.data; +package org.labkey.api.migration; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.labkey.api.data.DatabaseMigrationService.MigrationSchemaHandler; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; import java.util.Set; import java.util.function.Predicate; diff --git a/api/src/org/labkey/api/migration/DatabaseMigrationService.java b/api/src/org/labkey/api/migration/DatabaseMigrationService.java new file mode 100644 index 00000000000..dfcfa530610 --- /dev/null +++ b/api/src/org/labkey/api/migration/DatabaseMigrationService.java @@ -0,0 +1,83 @@ +package org.labkey.api.migration; + +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.SimpleFilter.FilterClause; +import org.labkey.api.data.TableInfo; +import org.labkey.api.migration.DatabaseMigrationConfiguration.DefaultDatabaseMigrationConfiguration; +import org.labkey.api.query.FieldKey; +import org.labkey.api.services.ServiceRegistry; +import org.labkey.api.util.ConfigurationException; +import org.labkey.api.util.GUID; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.vfs.FileLike; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public interface DatabaseMigrationService +{ + Logger LOG = LogHelper.getLogger(DatabaseMigrationService.class, "Information about database migration"); + + record DataFilter(Set containers, String column, FilterClause condition) {} + + static @NotNull DatabaseMigrationService get() + { + DatabaseMigrationService ret = ServiceRegistry.get().getService(DatabaseMigrationService.class); + return ret != null ? ret : new DatabaseMigrationService() {}; + } + + static void setInstance(DatabaseMigrationService impl) + { + ServiceRegistry.get().registerService(DatabaseMigrationService.class, impl); + } + + default DatabaseMigrationConfiguration getDatabaseMigrationConfiguration(FileLike labkeyRoot, @Nullable String migration) + { + return new DefaultDatabaseMigrationConfiguration(); + } + + // By default, no-op implementation that simply logs + default void migrate(DatabaseMigrationConfiguration configuration) + { + LOG.warn("Database migration service is not present; database migration is a premium feature."); + } + + // By default, no-op implementations + default void registerSchemaHandler(MigrationSchemaHandler schemaHandler) {} + default void registerMigrationFilter(MigrationFilter filter) {} + + default @Nullable MigrationFilter getMigrationFilter(String propertyName) + { + return null; + } + + default void copySourceTableToTargetTable(DatabaseMigrationConfiguration configuration, TableInfo sourceTable, TableInfo targetTable, DbSchemaType schemaType, boolean updateSequences, String additionalLogMessage, MigrationSchemaHandler schemaHandler) {} + default void updateSequences(TableInfo sourceTable, TableInfo targetTable) {} + + // Helper method that parses a data filter then adds it and its container to the provided collections, coalescing + // cases where multiple containers specify the same filter + static void addDataFilter(String filterName, List dataFilters, Set filteredContainers, GUID guid, String filter) + { + String[] filterParts = filter.split("="); + if (filterParts.length != 2) + throw new ConfigurationException("Bad " + filterName + " value; expected =: " + filter); + + if (!filteredContainers.add(guid)) + throw new ConfigurationException("Duplicate " + filterName + " entry for container " + guid); + + String column = filterParts[0]; + String value = filterParts[1]; + FilterClause clause = CompareType.EQUAL.createFilterClause(new FieldKey(null, column), value); + // If another container is already using this filter clause, then simply add this guid to that filter. + // Otherwise, add a new domain filter to the list. + dataFilters.stream() + .filter(df -> df.column().equals(column) && df.condition().equals(clause)) + .findFirst() + .ifPresentOrElse(df -> df.containers().add(guid), () -> dataFilters.add(new DataFilter(new HashSet<>(Set.of(guid)), filterParts[0], clause))); + } +} diff --git a/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java b/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java new file mode 100644 index 00000000000..1bdb831e86b --- /dev/null +++ b/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java @@ -0,0 +1,306 @@ +package org.labkey.api.migration; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.attachments.AttachmentType; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DatabaseTableType; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.ForeignKey; +import org.labkey.api.data.InClauseGenerator; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.TempTableInClauseGenerator; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.TableSorter; +import org.labkey.api.util.GUID; +import org.labkey.api.util.StringUtilsLabKey; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +public class DefaultMigrationSchemaHandler implements MigrationSchemaHandler +{ + private final DbSchema _schema; + + public DefaultMigrationSchemaHandler(DbSchema schema) + { + _schema = schema; + } + + @Override + public DbSchema getSchema() + { + return _schema; + } + + @Override + public void beforeVerification() + { + } + + @Override + public void beforeSchema() + { + } + + @Override + public List getTablesToCopy() + { + Set sortedTables = new LinkedHashSet<>(TableSorter.sort(getSchema(), true)); + + Set allTables = getSchema().getTableNames().stream() + .map(getSchema()::getTable) + .collect(Collectors.toCollection(HashSet::new)); + allTables.removeAll(sortedTables); + + if (!allTables.isEmpty()) + { + DatabaseMigrationService.LOG.info("These tables were removed by TableSorter: {}", allTables); + } + + return sortedTables.stream() + // Skip all views and virtual tables (e.g., test.Containers2, which is a table on SS but a view on PG) + .filter(table -> table.getTableType() == DatabaseTableType.TABLE) + .collect(Collectors.toCollection(ArrayList::new)); // Ensure mutable + } + + @Override + public SimpleFilter.FilterClause getTableFilterClause(TableInfo sourceTable, Set containers) + { + return getContainerClause(sourceTable, containers); + } + + @Override + public SimpleFilter.FilterClause getContainerClause(TableInfo sourceTable, Set containers) + { + FieldKey containerFieldKey = getContainerFieldKey(sourceTable); + + if (containerFieldKey == SITE_WIDE_TABLE) + return new SimpleFilter.SQLClause(new SQLFragment("TRUE")); + + return new SimpleFilter.InClause(containerFieldKey, containers); + } + + @Override + public @Nullable FieldKey getContainerFieldKey(TableInfo table) + { + FieldKey fKey = table.getContainerFieldKey(); + + if (fKey != null) + return fKey; + + for (ColumnInfo col : table.getColumns()) + { + ForeignKey fk = TableSorter.getForeignKey(table, col, true); + if (fk != null) + { + // Use the table's schema (or a migration schema retrieved from the table's scope), since we want a Migration schema with XML metadata applied + DbSchema tableSchema = table.getSchema(); + DbSchema lookupSchema = fk.getLookupSchemaKey().equals(new SchemaKey(null, tableSchema.getName())) ? + tableSchema : + tableSchema.getScope().getSchema(fk.getLookupSchemaName(), DbSchemaType.Migration); + TableInfo lookupTableInfo = lookupSchema.getTable(fk.getLookupTableName()); + if (lookupTableInfo != null) + { + fKey = lookupTableInfo.getContainerFieldKey(); + + if (null == fKey) + { + // Ignore self joins + if (!lookupTableInfo.getName().equalsIgnoreCase(table.getName())) + { + fKey = getContainerFieldKey(lookupTableInfo); + } + } + + if (fKey != null) + return FieldKey.fromParts(col.getFieldKey(), fKey); + } + } + } + + return null; + } + + @Override + public final SimpleFilter.FilterClause getDomainDataFilterClause(Set copyContainers, Set filteredContainers, List domainFilters, TableInfo sourceTable, Set selectColumnNames) + { + // Filtered case: remove the filtered containers from the unconditional container set + Set otherContainers = new HashSet<>(copyContainers); + otherContainers.removeAll(filteredContainers); + SimpleFilter.FilterClause ret = getContainerClause(sourceTable, otherContainers); + + SimpleFilter.OrClause orClause = new SimpleFilter.OrClause(); + + // Delegate to the MigrationSchemaHandler to add domain-filtered containers back with their special filter applied + domainFilters.forEach(filter -> addDomainDataFilterClause(orClause, filter, sourceTable, selectColumnNames)); + + if (!orClause.getClauses().isEmpty()) + { + orClause.addClause(ret); + ret = orClause; + } + + return ret; + } + + @Override + public void addDomainDataFilterClause(SimpleFilter.OrClause orClause, DatabaseMigrationService.DataFilter filter, TableInfo sourceTable, Set selectColumnNames) + { + addDataFilterClause(orClause, filter, sourceTable, selectColumnNames); + } + + // Add a filter and return true if the column exists directly on the table + protected boolean addDataFilterClause(SimpleFilter.OrClause orClause, DatabaseMigrationService.DataFilter filter, TableInfo sourceTable, Set selectColumnNames) + { + boolean columnExists = selectColumnNames.contains(filter.column()); + + if (columnExists) + { + // Select all rows in this domain-filtered container that meet its criteria + orClause.addClause( + new SimpleFilter.AndClause( + getContainerClause(sourceTable, filter.containers()), + filter.condition() + ) + ); + } + + return columnExists; + } + + // Add a clause that selects all rows where the object property with equals the filter value. This + // is only for provisioned tables that lack an ObjectId, MaterialId, or DataId column. + protected void addObjectPropertyClause(SimpleFilter.OrClause orClause, DatabaseMigrationService.DataFilter filter, TableInfo sourceTable, int propertyId) + { + SQLFragment flagWhere = new SQLFragment("lsid IN (SELECT ObjectURI FROM exp.Object o INNER JOIN exp.ObjectProperty op ON o.ObjectId = op.ObjectId WHERE StringValue = ? AND PropertyId = ?)", filter.condition().getParamVals()[0], propertyId); + + orClause.addClause( + new SimpleFilter.AndClause( + getContainerClause(sourceTable, filter.containers()), + new SimpleFilter.SQLClause(flagWhere) + ) + ); + } + + private Integer _commentPropertyId = null; + + protected synchronized int getCommentPropertyId(DbScope scope) + { + if (_commentPropertyId == null) + { + // Get the exp.PropertyDescriptor table from the source scope + TableInfo propertyDescriptor = scope.getSchema("exp", DbSchemaType.Migration).getTable("PropertyDescriptor"); + // Select the PropertyId associated with built-in Flag fields ("urn:exp.labkey.org/#Comment") + Integer propertyId = new TableSelector(propertyDescriptor, Collections.singleton("PropertyId"), new SimpleFilter(FieldKey.fromParts("PropertyURI"), "urn:exp.labkey.org/#Comment"), null).getObject(Integer.class); + if (propertyId == null) + throw new RuntimeException("PropertyDescriptor for built-in Flag field not found"); + else + _commentPropertyId = propertyId; + } + + return _commentPropertyId; + } + + protected String rowsNotCopied(int count) + { + return " " + StringUtilsLabKey.pluralize(count, "row") + " not copied"; + } + + @Override + public void afterTable(TableInfo sourceTable, TableInfo targetTable, SimpleFilter notCopiedFilter) + { + } + + @Override + public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema) + { + // Now that the target tables in this schema have been populated, copy all associated attachments. By + // default, use this handler's attachment types to select from the target tables all EntityIds that might be + // attachment parents (this avoids re-running potentially expensive queries on the source tables). Use the + // set of EntityIds to copy those attachments from the core.Documents table in the source database. Override + // if special behavior is required, for example, AttachmentTypes that use documentNameColumn since that + // requires querying and re-filtering the source tables instead. + getAttachmentTypes().forEach(type -> { + SQLFragment sql = type.getSelectParentEntityIdsSql(); + if (sql != null) + { + Collection entityIds = new SqlSelector(targetSchema, sql).getCollection(String.class); + SQLFragment selectParents = new SQLFragment("Parent"); + // This query against the source database is likely to contain a large IN clause, so use an alternative InClauseGenerator + sourceSchema.getSqlDialect().appendInClauseSql(selectParents, entityIds, getTempTableInClauseGenerator(sourceSchema.getScope())); + copyAttachments(configuration, sourceSchema, new SimpleFilter.SQLClause(selectParents), type); + } + + // TODO: implement remaining AttachmentTypes + // TODO: throw if some registered AttachmentType is not seen + // TODO: fail if type.getSelectParentEntityIdsSql() returns null + }); + } + + // Creates a TempTableInClauseGenerator that targets the *source* temp schema instead of the default + // DbSchema.getTemp(). Required for large IN clauses used against the source database. + protected InClauseGenerator getTempTableInClauseGenerator(DbScope sourceScope) + { + return new TempTableInClauseGenerator(() -> sourceScope.getSchema("temp", DbSchemaType.Bare)); + } + + private static final Set SEEN = new HashSet<>(); + + // Copy all core.Documents rows that match the provided filter clause + protected void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, SimpleFilter.FilterClause filterClause, AttachmentType... type) + { + SEEN.addAll(Arrays.asList(type)); + String additionalMessage = " associated with " + Arrays.stream(type).map(t -> t.getClass().getSimpleName()).collect(Collectors.joining(", ")); + TableInfo sourceDocumentsTable = sourceSchema.getScope().getSchema("core", DbSchemaType.Migration).getTable("Documents"); + TableInfo targetDocumentsTable = CoreSchema.getInstance().getTableInfoDocuments(); + DatabaseMigrationService.get().copySourceTableToTargetTable(configuration, sourceDocumentsTable, targetDocumentsTable, DbSchemaType.Module, false, additionalMessage, new DefaultMigrationSchemaHandler(CoreSchema.getInstance().getSchema()) + { + @Override + public SimpleFilter.FilterClause getTableFilterClause(TableInfo sourceTable, Set containers) + { + return filterClause; + } + }); + } + + public static void logUnseenAttachmentTypes() + { + Set unseen = new HashSet<>(AttachmentService.get().getAttachmentTypes()); + unseen.removeAll(SEEN); + + DatabaseMigrationService.LOG.info("These AttachmentTypes have not been seen: {}", unseen.stream().map(type -> type.getClass().getSimpleName()).collect(Collectors.joining(", "))); + } + + @Override + public @NotNull Collection getAttachmentTypes() + { + return List.of(); + } + + @Override + public void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema) + { + } + + @Override + public void afterMigration(DatabaseMigrationConfiguration configuration) + { + } +} diff --git a/api/src/org/labkey/api/migration/ExperimentDeleteService.java b/api/src/org/labkey/api/migration/ExperimentDeleteService.java new file mode 100644 index 00000000000..033eea366fa --- /dev/null +++ b/api/src/org/labkey/api/migration/ExperimentDeleteService.java @@ -0,0 +1,27 @@ +package org.labkey.api.migration; + +import org.jetbrains.annotations.NotNull; +import org.labkey.api.services.ServiceRegistry; + +import java.util.Collection; + +public interface ExperimentDeleteService +{ + static @NotNull ExperimentDeleteService get() + { + ExperimentDeleteService ret = ServiceRegistry.get().getService(ExperimentDeleteService.class); + if (ret == null) + throw new IllegalStateException("ExperimentDeleteService not found"); + return ret; + } + + static void setInstance(ExperimentDeleteService impl) + { + ServiceRegistry.get().registerService(ExperimentDeleteService.class, impl); + } + + /** + * Deletes all rows from exp.Data, exp.Object, and related tables associated with the provided ObjectIds + */ + void deleteDataRows(Collection objectIds); +} diff --git a/api/src/org/labkey/api/data/MigrationDbSchema.java b/api/src/org/labkey/api/migration/MigrationDbSchema.java similarity index 76% rename from api/src/org/labkey/api/data/MigrationDbSchema.java rename to api/src/org/labkey/api/migration/MigrationDbSchema.java index cd94dc17c01..4cba722d0c3 100644 --- a/api/src/org/labkey/api/data/MigrationDbSchema.java +++ b/api/src/org/labkey/api/migration/MigrationDbSchema.java @@ -1,5 +1,9 @@ -package org.labkey.api.data; +package org.labkey.api.migration; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.SchemaTableInfoFactory; import org.labkey.api.module.Module; import java.util.Map; diff --git a/api/src/org/labkey/api/migration/MigrationFilter.java b/api/src/org/labkey/api/migration/MigrationFilter.java new file mode 100644 index 00000000000..305fd747e9c --- /dev/null +++ b/api/src/org/labkey/api/migration/MigrationFilter.java @@ -0,0 +1,17 @@ +package org.labkey.api.migration; + +import org.jetbrains.annotations.Nullable; +import org.labkey.api.util.GUID; + +/** + * A MigrationFilter adds support for the named filter property in the migration configuration file. If present, + * saveFilter() is called with the container guid and property value. Modules can register these to present + * module-specific filters. + */ +public interface MigrationFilter +{ + String getName(); + + // Implementations should validate guid nullity + void saveFilter(@Nullable GUID guid, String value); +} diff --git a/api/src/org/labkey/api/migration/MigrationSchemaHandler.java b/api/src/org/labkey/api/migration/MigrationSchemaHandler.java new file mode 100644 index 00000000000..e2fd6159457 --- /dev/null +++ b/api/src/org/labkey/api/migration/MigrationSchemaHandler.java @@ -0,0 +1,58 @@ +package org.labkey.api.migration; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.attachments.AttachmentType; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.query.FieldKey; +import org.labkey.api.util.GUID; + +import java.util.Collection; +import java.util.List; +import java.util.Set; + +public interface MigrationSchemaHandler +{ + // Marker for tables to declare themselves as site-wide (no container filtering) + FieldKey SITE_WIDE_TABLE = FieldKey.fromParts("site-wide"); + + DbSchema getSchema(); + + void beforeVerification(); + + void beforeSchema(); + + List getTablesToCopy(); + + // Create a filter clause that selects from all specified containers and (in some overrides) applies table-specific filters + SimpleFilter.FilterClause getTableFilterClause(TableInfo sourceTable, Set containers); + + // Create a filter clause that selects from all specified containers + SimpleFilter.FilterClause getContainerClause(TableInfo sourceTable, Set containers); + + // Return the FieldKey that can be used to filter this table by container. Special values SITE_WIDE_TABLE and + // DUMMY_FIELD_KEY can be returned for special behaviors. DUMMY_FIELD_KEY ensures that the handler's custom + // getContainerClause() is always called. SITE_WIDE_TABLE is used to select all rows. + @Nullable FieldKey getContainerFieldKey(TableInfo sourceTable); + + // Create a filter clause that selects all rows from unfiltered containers plus filtered rows from the filtered containers + SimpleFilter.FilterClause getDomainDataFilterClause(Set copyContainers, Set filteredContainers, List domainFilters, TableInfo sourceTable, Set selectColumnNames); + + void addDomainDataFilterClause(SimpleFilter.OrClause orClause, DatabaseMigrationService.DataFilter filter, TableInfo sourceTable, Set selectColumnNames); + + // Do any necessary clean up after the target table has been populated. notCopiedFilter selects all rows in the + // source table that were NOT copied to the target table. (For example, rows in a global table not copied due to + // container filtering or rows in a provisioned table not copied due to domain data filtering.) + void afterTable(TableInfo sourceTable, TableInfo targetTable, SimpleFilter notCopiedFilter); + + void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema); + + // TODO: Return Collection, indicating which attachment types it handled? + void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema); + + @NotNull Collection getAttachmentTypes(); + + void afterMigration(DatabaseMigrationConfiguration configuration); +} diff --git a/api/src/org/labkey/api/module/ModuleLoader.java b/api/src/org/labkey/api/module/ModuleLoader.java index 1dfd255e7c0..49d5743d602 100644 --- a/api/src/org/labkey/api/module/ModuleLoader.java +++ b/api/src/org/labkey/api/module/ModuleLoader.java @@ -39,8 +39,6 @@ import org.labkey.api.data.Container; import org.labkey.api.data.ConvertHelper; import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DatabaseMigrationConfiguration; -import org.labkey.api.data.DatabaseMigrationService; import org.labkey.api.data.DatabaseTableType; import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbSchemaType; @@ -61,6 +59,8 @@ import org.labkey.api.data.TableSelector; import org.labkey.api.data.dialect.DatabaseNotSupportedException; import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.migration.DatabaseMigrationConfiguration; +import org.labkey.api.migration.DatabaseMigrationService; import org.labkey.api.module.ModuleUpgrader.Execution; import org.labkey.api.resource.Resource; import org.labkey.api.security.SecurityManager; diff --git a/assay/src/org/labkey/assay/AssayModule.java b/assay/src/org/labkey/assay/AssayModule.java index 3f34529b3be..df7bf953181 100644 --- a/assay/src/org/labkey/assay/AssayModule.java +++ b/assay/src/org/labkey/assay/AssayModule.java @@ -39,8 +39,6 @@ import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.ContainerType; -import org.labkey.api.data.DatabaseMigrationService; -import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationSchemaHandler; import org.labkey.api.data.TableInfo; import org.labkey.api.data.UpgradeCode; import org.labkey.api.data.generator.DataGeneratorRegistry; @@ -48,6 +46,8 @@ import org.labkey.api.exp.api.ExpProtocol; import org.labkey.api.exp.api.ExperimentService; import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.migration.DatabaseMigrationService; +import org.labkey.api.migration.DefaultMigrationSchemaHandler; import org.labkey.api.module.AdminLinkManager; import org.labkey.api.module.FolderTypeManager; import org.labkey.api.module.Module; diff --git a/assay/src/org/labkey/assay/AssayResultMigrationSchemaHandler.java b/assay/src/org/labkey/assay/AssayResultMigrationSchemaHandler.java index 218fc955ac6..7acaa367818 100644 --- a/assay/src/org/labkey/assay/AssayResultMigrationSchemaHandler.java +++ b/assay/src/org/labkey/assay/AssayResultMigrationSchemaHandler.java @@ -1,10 +1,8 @@ package org.labkey.assay; +import org.apache.commons.lang3.Strings; import org.apache.logging.log4j.Logger; import org.labkey.api.assay.AbstractTsvAssayProvider; -import org.labkey.api.data.DatabaseMigrationService.DataFilter; -import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationSchemaHandler; -import org.labkey.api.data.DatabaseMigrationService.ExperimentDeleteService; import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.SQLFragment; @@ -14,8 +12,12 @@ import org.labkey.api.data.SimpleFilter.SQLClause; import org.labkey.api.data.SqlSelector; import org.labkey.api.data.TableInfo; +import org.labkey.api.migration.DatabaseMigrationService.DataFilter; +import org.labkey.api.migration.DefaultMigrationSchemaHandler; +import org.labkey.api.migration.ExperimentDeleteService; import org.labkey.api.util.GUID; import org.labkey.api.util.logging.LogHelper; +import org.labkey.assay.plate.PlateReplicateStatsDomainKind; import java.util.Collection; import java.util.Set; @@ -29,18 +31,18 @@ public AssayResultMigrationSchemaHandler() super(DbSchema.get(AbstractTsvAssayProvider.ASSAY_SCHEMA_NAME, DbSchemaType.Provisioned)); } - // Provisioned assay result tables occasionally have no DataId column; hopefully they have an LSID column. - private boolean hasDataIdColumn(TableInfo sourceTable) + private boolean skipTable(TableInfo sourceTable) { - return sourceTable.getColumn("DataId") != null; + // For now, we're ignoring this table since it's empty in our first migration client's database + return Strings.CI.endsWith(sourceTable.getName(), PlateReplicateStatsDomainKind.ASSAY_PLATE_REPLICATE); } @Override public FilterClause getContainerClause(TableInfo sourceTable, Set containers) { - return new SQLClause( - new SQLFragment(hasDataIdColumn(sourceTable) ? "DataId IN (SELECT RowId" : "LSID IN (SELECT LSID") - .append(" FROM exp.Data WHERE Container") + return new SQLClause(skipTable(sourceTable) ? + new SQLFragment("1 = 0") : + new SQLFragment("DataId IN (SELECT RowId FROM exp.Data WHERE Container") .appendInClause(containers, sourceTable.getSqlDialect()) .append(")") ); @@ -55,26 +57,27 @@ public void addDomainDataFilterClause(OrClause orClause, DataFilter filter, Tabl @Override public void afterTable(TableInfo sourceTable, TableInfo targetTable, SimpleFilter notCopiedFilter) { - SQLFragment objectIdSql = new SQLFragment("SELECT ObjectId FROM exp.Data WHERE ") - .append(hasDataIdColumn(sourceTable) ? "RowId IN (SELECT DataId" : "LSID IN (SELECT LSID") - .append(" FROM ") - .appendIdentifier(sourceTable.getSelectName()) - .append(" ") - .append(notCopiedFilter.getSQLFragment(sourceTable.getSqlDialect())) - .append(")"); + if (!skipTable(sourceTable)) + { + SQLFragment objectIdSql = new SQLFragment("SELECT ObjectId FROM exp.Data WHERE RowId IN (SELECT DataId FROM ") + .appendIdentifier(sourceTable.getSelectName()) + .append(" ") + .append(notCopiedFilter.getSQLFragment(sourceTable.getSqlDialect())) + .append(")"); - Collection notCopiedObjectIds = new SqlSelector(sourceTable.getSchema(), objectIdSql).getCollection(Long.class); + Collection notCopiedObjectIds = new SqlSelector(sourceTable.getSchema(), objectIdSql).getCollection(Long.class); - if (notCopiedObjectIds.isEmpty()) - { - LOG.info(rowsNotCopied(0)); - } - else - { - LOG.info("{} -- deleting associated rows from exp.Data, exp.Object, etc.", rowsNotCopied(notCopiedObjectIds.size())); + if (notCopiedObjectIds.isEmpty()) + { + LOG.info(rowsNotCopied(0)); + } + else + { + LOG.info("{} -- deleting associated rows from exp.Data, exp.Object, etc.", rowsNotCopied(notCopiedObjectIds.size())); - // Delete exp.Data, exp.Object, etc. rows associated with the rows that weren't copied - ExperimentDeleteService.get().deleteDataRows(notCopiedObjectIds); + // Delete exp.Data, exp.Object, etc. rows associated with the rows that weren't copied + ExperimentDeleteService.get().deleteDataRows(notCopiedObjectIds); + } } } } diff --git a/core/src/org/labkey/core/CoreMigrationSchemaHandler.java b/core/src/org/labkey/core/CoreMigrationSchemaHandler.java index c137724b2a8..603bda092ec 100644 --- a/core/src/org/labkey/core/CoreMigrationSchemaHandler.java +++ b/core/src/org/labkey/core/CoreMigrationSchemaHandler.java @@ -7,9 +7,6 @@ import org.labkey.api.data.CompareType; import org.labkey.api.data.CompareType.CompareClause; import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DatabaseMigrationConfiguration; -import org.labkey.api.data.DatabaseMigrationService; -import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationSchemaHandler; import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.DbScope; @@ -23,6 +20,11 @@ import org.labkey.api.data.Table; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TestSchema; +import org.labkey.api.files.FileSystemAttachmentType; +import org.labkey.api.migration.DatabaseMigrationConfiguration; +import org.labkey.api.migration.DatabaseMigrationService; +import org.labkey.api.migration.DefaultMigrationSchemaHandler; +import org.labkey.api.migration.MigrationFilter; import org.labkey.api.module.ModuleLoader; import org.labkey.api.query.FieldKey; import org.labkey.api.reports.report.ReportType; @@ -35,7 +37,7 @@ import java.util.List; import java.util.Set; -class CoreMigrationSchemaHandler extends DefaultMigrationSchemaHandler implements DatabaseMigrationService.MigrationFilter +class CoreMigrationSchemaHandler extends DefaultMigrationSchemaHandler implements MigrationFilter { static void register() { @@ -207,6 +209,7 @@ public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSche return List.of( AuthenticationLogoType.get(), AvatarType.get(), + FileSystemAttachmentType.get(), ReportType.get() ); } diff --git a/experiment/src/org/labkey/experiment/DataClassMigrationSchemaHandler.java b/experiment/src/org/labkey/experiment/DataClassMigrationSchemaHandler.java index 79793151405..45232046c93 100644 --- a/experiment/src/org/labkey/experiment/DataClassMigrationSchemaHandler.java +++ b/experiment/src/org/labkey/experiment/DataClassMigrationSchemaHandler.java @@ -4,11 +4,6 @@ import org.jetbrains.annotations.NotNull; import org.labkey.api.attachments.AttachmentType; import org.labkey.api.collections.Sets; -import org.labkey.api.data.DatabaseMigrationConfiguration; -import org.labkey.api.data.DatabaseMigrationService; -import org.labkey.api.data.DatabaseMigrationService.DataFilter; -import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationSchemaHandler; -import org.labkey.api.data.DatabaseMigrationService.ExperimentDeleteService; import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.DbScope; @@ -24,6 +19,11 @@ import org.labkey.api.data.TableSelector; import org.labkey.api.data.dialect.SqlDialect; import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.migration.DatabaseMigrationConfiguration; +import org.labkey.api.migration.DatabaseMigrationService; +import org.labkey.api.migration.DatabaseMigrationService.DataFilter; +import org.labkey.api.migration.DefaultMigrationSchemaHandler; +import org.labkey.api.migration.ExperimentDeleteService; import org.labkey.api.query.FieldKey; import org.labkey.api.util.GUID; import org.labkey.api.util.StringUtilsLabKey; diff --git a/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java b/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java index ce5285694ef..7bc86ef411c 100644 --- a/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java +++ b/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java @@ -6,8 +6,6 @@ import org.labkey.api.attachments.AttachmentType; import org.labkey.api.data.CompareType; import org.labkey.api.data.CompareType.CompareClause; -import org.labkey.api.data.DatabaseMigrationConfiguration; -import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationSchemaHandler; import org.labkey.api.data.DbSchema; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter.AndClause; @@ -20,6 +18,8 @@ import org.labkey.api.exp.OntologyManager; import org.labkey.api.exp.api.ExpProtocolAttachmentType; import org.labkey.api.exp.api.ExpRunAttachmentType; +import org.labkey.api.migration.DatabaseMigrationConfiguration; +import org.labkey.api.migration.DefaultMigrationSchemaHandler; import org.labkey.api.query.FieldKey; import org.labkey.api.util.GUID; import org.labkey.api.util.logging.LogHelper; diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index c64f5ea669b..d37893fba01 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -30,7 +30,6 @@ import org.labkey.api.data.ContainerFilter; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DatabaseMigrationService; import org.labkey.api.data.DbSchema; import org.labkey.api.data.JdbcType; import org.labkey.api.data.NameGenerator; @@ -73,6 +72,9 @@ import org.labkey.api.exp.xar.LsidUtils; import org.labkey.api.files.FileContentService; import org.labkey.api.files.TableUpdaterFileListener; +import org.labkey.api.migration.AssaySkipFilter; +import org.labkey.api.migration.DatabaseMigrationService; +import org.labkey.api.migration.ExperimentDeleteService; import org.labkey.api.module.ModuleContext; import org.labkey.api.module.ModuleLoader; import org.labkey.api.module.SpringModule; @@ -877,7 +879,8 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM DatabaseMigrationService.get().registerSchemaHandler(new SampleTypeMigrationSchemaHandler()); DataClassMigrationSchemaHandler dcHandler = new DataClassMigrationSchemaHandler(); DatabaseMigrationService.get().registerSchemaHandler(dcHandler); - DatabaseMigrationService.ExperimentDeleteService.setInstance(dcHandler); + ExperimentDeleteService.setInstance(dcHandler); + DatabaseMigrationService.get().registerMigrationFilter(new AssaySkipFilter()); } @Override diff --git a/experiment/src/org/labkey/experiment/SampleTypeMigrationSchemaHandler.java b/experiment/src/org/labkey/experiment/SampleTypeMigrationSchemaHandler.java index 691cf334991..4c2d2f89d42 100644 --- a/experiment/src/org/labkey/experiment/SampleTypeMigrationSchemaHandler.java +++ b/experiment/src/org/labkey/experiment/SampleTypeMigrationSchemaHandler.java @@ -1,8 +1,6 @@ package org.labkey.experiment; import org.apache.logging.log4j.Logger; -import org.labkey.api.data.DatabaseMigrationService.DataFilter; -import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationSchemaHandler; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.SimpleFilter.FilterClause; @@ -14,6 +12,8 @@ import org.labkey.api.data.dialect.SqlDialect; import org.labkey.api.exp.OntologyManager; import org.labkey.api.exp.api.SampleTypeDomainKind; +import org.labkey.api.migration.DatabaseMigrationService.DataFilter; +import org.labkey.api.migration.DefaultMigrationSchemaHandler; import org.labkey.api.util.GUID; import org.labkey.api.util.logging.LogHelper; diff --git a/filecontent/src/org/labkey/filecontent/FileContentModule.java b/filecontent/src/org/labkey/filecontent/FileContentModule.java index 5b79bcfdd03..a4dfb2e733a 100644 --- a/filecontent/src/org/labkey/filecontent/FileContentModule.java +++ b/filecontent/src/org/labkey/filecontent/FileContentModule.java @@ -28,6 +28,7 @@ import org.labkey.api.data.TableInfo; import org.labkey.api.exp.property.PropertyService; import org.labkey.api.files.FileContentService; +import org.labkey.api.files.FileSystemAttachmentType; import org.labkey.api.files.view.FilesWebPart; import org.labkey.api.message.digest.DailyMessageDigest; import org.labkey.api.message.settings.MessageConfigService; diff --git a/filecontent/src/org/labkey/filecontent/FileSystemAttachmentParent.java b/filecontent/src/org/labkey/filecontent/FileSystemAttachmentParent.java index e425e51da14..0d7859a9767 100644 --- a/filecontent/src/org/labkey/filecontent/FileSystemAttachmentParent.java +++ b/filecontent/src/org/labkey/filecontent/FileSystemAttachmentParent.java @@ -26,6 +26,7 @@ import org.labkey.api.attachments.AttachmentType; import org.labkey.api.data.Container; import org.labkey.api.files.FileContentService; +import org.labkey.api.files.FileSystemAttachmentType; import org.labkey.api.security.User; import org.labkey.api.util.FileUtil; @@ -247,7 +248,6 @@ public void deleteAttachment(User user, @Nullable String name) LOG.warn(e.getMessage()); } } - }); } } diff --git a/issues/src/org/labkey/issue/IssueMigrationSchemaHandler.java b/issues/src/org/labkey/issue/IssueMigrationSchemaHandler.java index ee5f8d48eb6..cc71ce1bca8 100644 --- a/issues/src/org/labkey/issue/IssueMigrationSchemaHandler.java +++ b/issues/src/org/labkey/issue/IssueMigrationSchemaHandler.java @@ -4,8 +4,6 @@ import org.jetbrains.annotations.NotNull; import org.labkey.api.attachments.AttachmentType; import org.labkey.api.collections.CsvSet; -import org.labkey.api.data.DatabaseMigrationConfiguration; -import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationSchemaHandler; import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.SQLFragment; @@ -16,6 +14,8 @@ import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.issues.IssuesSchema; +import org.labkey.api.migration.DatabaseMigrationConfiguration; +import org.labkey.api.migration.DefaultMigrationSchemaHandler; import org.labkey.api.query.FieldKey; import org.labkey.api.util.StringUtilsLabKey; import org.labkey.api.util.logging.LogHelper; diff --git a/issues/src/org/labkey/issue/IssuesModule.java b/issues/src/org/labkey/issue/IssuesModule.java index 64591419cf5..51ad08c0d1a 100644 --- a/issues/src/org/labkey/issue/IssuesModule.java +++ b/issues/src/org/labkey/issue/IssuesModule.java @@ -22,13 +22,13 @@ import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.DataRegion; -import org.labkey.api.data.DatabaseMigrationService; import org.labkey.api.data.SqlExecutor; import org.labkey.api.data.SqlSelector; import org.labkey.api.exp.property.PropertyService; import org.labkey.api.issues.IssueService; import org.labkey.api.issues.IssuesListDefService; import org.labkey.api.issues.IssuesSchema; +import org.labkey.api.migration.DatabaseMigrationService; import org.labkey.api.module.DefaultModule; import org.labkey.api.module.ModuleContext; import org.labkey.api.query.QueryService; diff --git a/list/src/org/labkey/list/ListModule.java b/list/src/org/labkey/list/ListModule.java index cf1ae836a0b..97e96f1220b 100644 --- a/list/src/org/labkey/list/ListModule.java +++ b/list/src/org/labkey/list/ListModule.java @@ -22,8 +22,6 @@ import org.labkey.api.attachments.AttachmentType; import org.labkey.api.audit.AuditLogService; import org.labkey.api.data.Container; -import org.labkey.api.data.DatabaseMigrationService; -import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationSchemaHandler; import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.SqlSelector; @@ -33,6 +31,8 @@ import org.labkey.api.exp.property.PropertyService; import org.labkey.api.lists.permissions.DesignListPermission; import org.labkey.api.lists.permissions.ManagePicklistsPermission; +import org.labkey.api.migration.DatabaseMigrationService; +import org.labkey.api.migration.DefaultMigrationSchemaHandler; import org.labkey.api.module.AdminLinkManager; import org.labkey.api.module.ModuleContext; import org.labkey.api.module.SpringModule; diff --git a/search/src/org/labkey/search/SearchModule.java b/search/src/org/labkey/search/SearchModule.java index 51dde323596..f4f9a3a7603 100644 --- a/search/src/org/labkey/search/SearchModule.java +++ b/search/src/org/labkey/search/SearchModule.java @@ -23,14 +23,14 @@ import org.labkey.api.cache.CacheManager; import org.labkey.api.data.ContainerFilter; import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DatabaseMigrationService; -import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationSchemaHandler; import org.labkey.api.data.DbSchema; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.data.UpgradeCode; import org.labkey.api.mbean.LabKeyManagement; import org.labkey.api.mbean.SearchMXBean; +import org.labkey.api.migration.DatabaseMigrationService; +import org.labkey.api.migration.DefaultMigrationSchemaHandler; import org.labkey.api.module.DefaultModule; import org.labkey.api.module.ModuleContext; import org.labkey.api.module.ModuleLoader; diff --git a/study/src/org/labkey/study/StudyModule.java b/study/src/org/labkey/study/StudyModule.java index 2d5358dd782..8df47d6aa96 100644 --- a/study/src/org/labkey/study/StudyModule.java +++ b/study/src/org/labkey/study/StudyModule.java @@ -29,8 +29,6 @@ import org.labkey.api.audit.AuditLogService; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DatabaseMigrationService; -import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationSchemaHandler; import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.PropertySchema; @@ -47,6 +45,8 @@ import org.labkey.api.files.FileContentService; import org.labkey.api.files.TableUpdaterFileListener; import org.labkey.api.message.digest.ReportAndDatasetChangeDigestProvider; +import org.labkey.api.migration.DatabaseMigrationService; +import org.labkey.api.migration.DefaultMigrationSchemaHandler; import org.labkey.api.module.AdminLinkManager; import org.labkey.api.module.DefaultFolderType; import org.labkey.api.module.FolderTypeManager; From d41e9d55af773fe37da9cd0860f03ac47a443e69 Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Tue, 11 Nov 2025 14:43:04 -0800 Subject: [PATCH 11/12] Finish attachment handling. Begin implementing AssaySkipContainers handling. --- .../api/migration/AssaySkipContainers.java | 26 +++++++ .../labkey/api/migration/AssaySkipFilter.java | 33 --------- .../DatabaseMigrationConfiguration.java | 45 ++---------- .../migration/DatabaseMigrationService.java | 3 +- ...DefaultDatabaseMigrationConfiguration.java | 50 +++++++++++++ .../DefaultMigrationSchemaHandler.java | 72 ++++++++++--------- .../api/migration/MigrationSchemaHandler.java | 14 ++-- .../AssayResultMigrationSchemaHandler.java | 34 +++++++-- .../core/CoreMigrationSchemaHandler.java | 14 ++-- .../ExperimentMigrationSchemaHandler.java | 18 +++++ .../labkey/experiment/ExperimentModule.java | 2 - 11 files changed, 184 insertions(+), 127 deletions(-) create mode 100644 api/src/org/labkey/api/migration/AssaySkipContainers.java delete mode 100644 api/src/org/labkey/api/migration/AssaySkipFilter.java create mode 100644 api/src/org/labkey/api/migration/DefaultDatabaseMigrationConfiguration.java diff --git a/api/src/org/labkey/api/migration/AssaySkipContainers.java b/api/src/org/labkey/api/migration/AssaySkipContainers.java new file mode 100644 index 00000000000..ca688510ab7 --- /dev/null +++ b/api/src/org/labkey/api/migration/AssaySkipContainers.java @@ -0,0 +1,26 @@ +package org.labkey.api.migration; + +import org.labkey.api.util.GUID; + +import java.util.Set; +import java.util.concurrent.CopyOnWriteArraySet; + +// Need to make the assay-skip containers available to both experiment and assay +public class AssaySkipContainers +{ + private static final Set SKIP_CONTAINERS = new CopyOnWriteArraySet<>(); + + private AssaySkipContainers() + { + } + + public static void addContainers(Set containers) + { + SKIP_CONTAINERS.addAll(containers); + } + + public static Set getContainers() + { + return SKIP_CONTAINERS; + } +} diff --git a/api/src/org/labkey/api/migration/AssaySkipFilter.java b/api/src/org/labkey/api/migration/AssaySkipFilter.java deleted file mode 100644 index 1f0b7c1e3dd..00000000000 --- a/api/src/org/labkey/api/migration/AssaySkipFilter.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.labkey.api.migration; - -import org.jetbrains.annotations.Nullable; -import org.labkey.api.util.ConfigurationException; -import org.labkey.api.util.GUID; - -import java.util.HashSet; -import java.util.Set; - -public class AssaySkipFilter implements MigrationFilter -{ - private static final Set SKIP_CONTAINERS = new HashSet<>(); - - @Override - public String getName() - { - return "AssaySkipFilter"; - } - - @Override - public void saveFilter(@Nullable GUID guid, String value) - { - if (null == guid) - throw new ConfigurationException(getName() + " must specify a GUID"); - - SKIP_CONTAINERS.add(guid); - } - - public static Set getSkipContainers() - { - return SKIP_CONTAINERS; - } -} diff --git a/api/src/org/labkey/api/migration/DatabaseMigrationConfiguration.java b/api/src/org/labkey/api/migration/DatabaseMigrationConfiguration.java index 1d92782419b..9d0a549cb12 100644 --- a/api/src/org/labkey/api/migration/DatabaseMigrationConfiguration.java +++ b/api/src/org/labkey/api/migration/DatabaseMigrationConfiguration.java @@ -2,6 +2,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.DbScope; import org.labkey.api.data.TableInfo; @@ -13,50 +14,12 @@ public interface DatabaseMigrationConfiguration { boolean shouldInsertData(); - default void beforeMigration(){}; + default void beforeMigration(){} DbScope getSourceScope(); DbScope getTargetScope(); @NotNull Set getSkipSchemas(); Predicate getColumnNameFilter(); @Nullable TableSelector getTableSelector(DbSchemaType schemaType, TableInfo sourceTable, TableInfo targetTable, Set selectColumnNames, MigrationSchemaHandler schemaHandler); - default void afterMigration(){}; - - class DefaultDatabaseMigrationConfiguration implements DatabaseMigrationConfiguration - { - @Override - public boolean shouldInsertData() - { - return true; - } - - @Override - public @Nullable DbScope getSourceScope() - { - return null; - } - - @Override - public DbScope getTargetScope() - { - return null; - } - - @Override - public @NotNull Set getSkipSchemas() - { - return Set.of(); - } - - @Override - public Predicate getColumnNameFilter() - { - return null; - } - - @Override - public TableSelector getTableSelector(DbSchemaType schemaType, TableInfo sourceTable, TableInfo targetTable, Set selectColumnNames, MigrationSchemaHandler schemaHandler) - { - return null; - } - } + default void copyAttachments(DbSchema sourceSchema, DbSchema targetSchema, MigrationSchemaHandler schemaHandler){} + default void afterMigration(){} } diff --git a/api/src/org/labkey/api/migration/DatabaseMigrationService.java b/api/src/org/labkey/api/migration/DatabaseMigrationService.java index dfcfa530610..ec2c0760a7c 100644 --- a/api/src/org/labkey/api/migration/DatabaseMigrationService.java +++ b/api/src/org/labkey/api/migration/DatabaseMigrationService.java @@ -7,7 +7,6 @@ import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.SimpleFilter.FilterClause; import org.labkey.api.data.TableInfo; -import org.labkey.api.migration.DatabaseMigrationConfiguration.DefaultDatabaseMigrationConfiguration; import org.labkey.api.query.FieldKey; import org.labkey.api.services.ServiceRegistry; import org.labkey.api.util.ConfigurationException; @@ -61,7 +60,7 @@ default void updateSequences(TableInfo sourceTable, TableInfo targetTable) {} // Helper method that parses a data filter then adds it and its container to the provided collections, coalescing // cases where multiple containers specify the same filter - static void addDataFilter(String filterName, List dataFilters, Set filteredContainers, GUID guid, String filter) + static void addDataFilter(String filterName, List dataFilters, Set filteredContainers, @NotNull GUID guid, String filter) { String[] filterParts = filter.split("="); if (filterParts.length != 2) diff --git a/api/src/org/labkey/api/migration/DefaultDatabaseMigrationConfiguration.java b/api/src/org/labkey/api/migration/DefaultDatabaseMigrationConfiguration.java new file mode 100644 index 00000000000..7dea199479d --- /dev/null +++ b/api/src/org/labkey/api/migration/DefaultDatabaseMigrationConfiguration.java @@ -0,0 +1,50 @@ +package org.labkey.api.migration; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; + +import java.util.Set; +import java.util.function.Predicate; + +public class DefaultDatabaseMigrationConfiguration implements DatabaseMigrationConfiguration +{ + @Override + public boolean shouldInsertData() + { + return true; + } + + @Override + public @Nullable DbScope getSourceScope() + { + return null; + } + + @Override + public DbScope getTargetScope() + { + return null; + } + + @Override + public @NotNull Set getSkipSchemas() + { + return Set.of(); + } + + @Override + public Predicate getColumnNameFilter() + { + return null; + } + + @Override + public TableSelector getTableSelector(DbSchemaType schemaType, TableInfo sourceTable, TableInfo targetTable, Set selectColumnNames, MigrationSchemaHandler schemaHandler) + { + return null; + } +} diff --git a/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java b/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java index 1bdb831e86b..019d5f6e30d 100644 --- a/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java +++ b/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java @@ -14,10 +14,16 @@ import org.labkey.api.data.InClauseGenerator; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SimpleFilter.AndClause; +import org.labkey.api.data.SimpleFilter.FilterClause; +import org.labkey.api.data.SimpleFilter.InClause; +import org.labkey.api.data.SimpleFilter.OrClause; +import org.labkey.api.data.SimpleFilter.SQLClause; import org.labkey.api.data.SqlSelector; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.data.TempTableInClauseGenerator; +import org.labkey.api.migration.DatabaseMigrationService.DataFilter; import org.labkey.api.query.FieldKey; import org.labkey.api.query.SchemaKey; import org.labkey.api.query.TableSorter; @@ -65,8 +71,8 @@ public List getTablesToCopy() Set sortedTables = new LinkedHashSet<>(TableSorter.sort(getSchema(), true)); Set allTables = getSchema().getTableNames().stream() - .map(getSchema()::getTable) - .collect(Collectors.toCollection(HashSet::new)); + .map(getSchema()::getTable) + .collect(Collectors.toCollection(HashSet::new)); allTables.removeAll(sortedTables); if (!allTables.isEmpty()) @@ -75,26 +81,26 @@ public List getTablesToCopy() } return sortedTables.stream() - // Skip all views and virtual tables (e.g., test.Containers2, which is a table on SS but a view on PG) - .filter(table -> table.getTableType() == DatabaseTableType.TABLE) - .collect(Collectors.toCollection(ArrayList::new)); // Ensure mutable + // Skip all views and virtual tables (e.g., test.Containers2, which is a table on SS but a view on PG) + .filter(table -> table.getTableType() == DatabaseTableType.TABLE) + .collect(Collectors.toCollection(ArrayList::new)); // Ensure mutable } @Override - public SimpleFilter.FilterClause getTableFilterClause(TableInfo sourceTable, Set containers) + public FilterClause getTableFilterClause(TableInfo sourceTable, Set containers) { return getContainerClause(sourceTable, containers); } @Override - public SimpleFilter.FilterClause getContainerClause(TableInfo sourceTable, Set containers) + public FilterClause getContainerClause(TableInfo sourceTable, Set containers) { FieldKey containerFieldKey = getContainerFieldKey(sourceTable); if (containerFieldKey == SITE_WIDE_TABLE) - return new SimpleFilter.SQLClause(new SQLFragment("TRUE")); + return new SQLClause(new SQLFragment("TRUE")); - return new SimpleFilter.InClause(containerFieldKey, containers); + return new InClause(containerFieldKey, containers); } @Override @@ -113,8 +119,8 @@ public SimpleFilter.FilterClause getContainerClause(TableInfo sourceTable, Set copyContainers, Set filteredContainers, List domainFilters, TableInfo sourceTable, Set selectColumnNames) + public final FilterClause getDomainDataFilterClause(Set copyContainers, Set filteredContainers, List domainFilters, TableInfo sourceTable, Set selectColumnNames) { // Filtered case: remove the filtered containers from the unconditional container set Set otherContainers = new HashSet<>(copyContainers); otherContainers.removeAll(filteredContainers); - SimpleFilter.FilterClause ret = getContainerClause(sourceTable, otherContainers); + FilterClause ret = getContainerClause(sourceTable, otherContainers); - SimpleFilter.OrClause orClause = new SimpleFilter.OrClause(); + OrClause orClause = new OrClause(); // Delegate to the MigrationSchemaHandler to add domain-filtered containers back with their special filter applied domainFilters.forEach(filter -> addDomainDataFilterClause(orClause, filter, sourceTable, selectColumnNames)); @@ -161,13 +167,13 @@ public final SimpleFilter.FilterClause getDomainDataFilterClause(Set copyC } @Override - public void addDomainDataFilterClause(SimpleFilter.OrClause orClause, DatabaseMigrationService.DataFilter filter, TableInfo sourceTable, Set selectColumnNames) + public void addDomainDataFilterClause(OrClause orClause, DataFilter filter, TableInfo sourceTable, Set selectColumnNames) { addDataFilterClause(orClause, filter, sourceTable, selectColumnNames); } // Add a filter and return true if the column exists directly on the table - protected boolean addDataFilterClause(SimpleFilter.OrClause orClause, DatabaseMigrationService.DataFilter filter, TableInfo sourceTable, Set selectColumnNames) + protected boolean addDataFilterClause(OrClause orClause, DataFilter filter, TableInfo sourceTable, Set selectColumnNames) { boolean columnExists = selectColumnNames.contains(filter.column()); @@ -175,10 +181,10 @@ protected boolean addDataFilterClause(SimpleFilter.OrClause orClause, DatabaseMi { // Select all rows in this domain-filtered container that meet its criteria orClause.addClause( - new SimpleFilter.AndClause( - getContainerClause(sourceTable, filter.containers()), - filter.condition() - ) + new AndClause( + getContainerClause(sourceTable, filter.containers()), + filter.condition() + ) ); } @@ -187,15 +193,15 @@ protected boolean addDataFilterClause(SimpleFilter.OrClause orClause, DatabaseMi // Add a clause that selects all rows where the object property with equals the filter value. This // is only for provisioned tables that lack an ObjectId, MaterialId, or DataId column. - protected void addObjectPropertyClause(SimpleFilter.OrClause orClause, DatabaseMigrationService.DataFilter filter, TableInfo sourceTable, int propertyId) + protected void addObjectPropertyClause(OrClause orClause, DataFilter filter, TableInfo sourceTable, int propertyId) { SQLFragment flagWhere = new SQLFragment("lsid IN (SELECT ObjectURI FROM exp.Object o INNER JOIN exp.ObjectProperty op ON o.ObjectId = op.ObjectId WHERE StringValue = ? AND PropertyId = ?)", filter.condition().getParamVals()[0], propertyId); orClause.addClause( - new SimpleFilter.AndClause( - getContainerClause(sourceTable, filter.containers()), - new SimpleFilter.SQLClause(flagWhere) - ) + new AndClause( + getContainerClause(sourceTable, filter.containers()), + new SQLClause(flagWhere) + ) ); } @@ -229,7 +235,7 @@ public void afterTable(TableInfo sourceTable, TableInfo targetTable, SimpleFilte } @Override - public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema) + public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema, Set copyContainers) { // Now that the target tables in this schema have been populated, copy all associated attachments. By // default, use this handler's attachment types to select from the target tables all EntityIds that might be @@ -245,12 +251,11 @@ public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSche SQLFragment selectParents = new SQLFragment("Parent"); // This query against the source database is likely to contain a large IN clause, so use an alternative InClauseGenerator sourceSchema.getSqlDialect().appendInClauseSql(selectParents, entityIds, getTempTableInClauseGenerator(sourceSchema.getScope())); - copyAttachments(configuration, sourceSchema, new SimpleFilter.SQLClause(selectParents), type); + copyAttachments(configuration, sourceSchema, new SQLClause(selectParents), type); } - // TODO: implement remaining AttachmentTypes + // TODO: fail if type.getSelectParentEntityIdsSql() returns null? // TODO: throw if some registered AttachmentType is not seen - // TODO: fail if type.getSelectParentEntityIdsSql() returns null }); } @@ -264,7 +269,7 @@ protected InClauseGenerator getTempTableInClauseGenerator(DbScope sourceScope) private static final Set SEEN = new HashSet<>(); // Copy all core.Documents rows that match the provided filter clause - protected void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, SimpleFilter.FilterClause filterClause, AttachmentType... type) + protected void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, FilterClause filterClause, AttachmentType... type) { SEEN.addAll(Arrays.asList(type)); String additionalMessage = " associated with " + Arrays.stream(type).map(t -> t.getClass().getSimpleName()).collect(Collectors.joining(", ")); @@ -273,7 +278,7 @@ protected void copyAttachments(DatabaseMigrationConfiguration configuration, DbS DatabaseMigrationService.get().copySourceTableToTargetTable(configuration, sourceDocumentsTable, targetDocumentsTable, DbSchemaType.Module, false, additionalMessage, new DefaultMigrationSchemaHandler(CoreSchema.getInstance().getSchema()) { @Override - public SimpleFilter.FilterClause getTableFilterClause(TableInfo sourceTable, Set containers) + public FilterClause getTableFilterClause(TableInfo sourceTable, Set containers) { return filterClause; } @@ -285,7 +290,10 @@ public static void logUnseenAttachmentTypes() Set unseen = new HashSet<>(AttachmentService.get().getAttachmentTypes()); unseen.removeAll(SEEN); - DatabaseMigrationService.LOG.info("These AttachmentTypes have not been seen: {}", unseen.stream().map(type -> type.getClass().getSimpleName()).collect(Collectors.joining(", "))); + if (SEEN.isEmpty()) + DatabaseMigrationService.LOG.info("All AttachmentTypes have been seen"); + else + DatabaseMigrationService.LOG.info("These AttachmentTypes have not been seen: {}", unseen.stream().map(type -> type.getClass().getSimpleName()).collect(Collectors.joining(", "))); } @Override diff --git a/api/src/org/labkey/api/migration/MigrationSchemaHandler.java b/api/src/org/labkey/api/migration/MigrationSchemaHandler.java index e2fd6159457..daa6aaa02e3 100644 --- a/api/src/org/labkey/api/migration/MigrationSchemaHandler.java +++ b/api/src/org/labkey/api/migration/MigrationSchemaHandler.java @@ -5,7 +5,10 @@ import org.labkey.api.attachments.AttachmentType; import org.labkey.api.data.DbSchema; import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SimpleFilter.FilterClause; +import org.labkey.api.data.SimpleFilter.OrClause; import org.labkey.api.data.TableInfo; +import org.labkey.api.migration.DatabaseMigrationService.DataFilter; import org.labkey.api.query.FieldKey; import org.labkey.api.util.GUID; @@ -27,10 +30,10 @@ public interface MigrationSchemaHandler List getTablesToCopy(); // Create a filter clause that selects from all specified containers and (in some overrides) applies table-specific filters - SimpleFilter.FilterClause getTableFilterClause(TableInfo sourceTable, Set containers); + FilterClause getTableFilterClause(TableInfo sourceTable, Set containers); // Create a filter clause that selects from all specified containers - SimpleFilter.FilterClause getContainerClause(TableInfo sourceTable, Set containers); + FilterClause getContainerClause(TableInfo sourceTable, Set containers); // Return the FieldKey that can be used to filter this table by container. Special values SITE_WIDE_TABLE and // DUMMY_FIELD_KEY can be returned for special behaviors. DUMMY_FIELD_KEY ensures that the handler's custom @@ -38,9 +41,9 @@ public interface MigrationSchemaHandler @Nullable FieldKey getContainerFieldKey(TableInfo sourceTable); // Create a filter clause that selects all rows from unfiltered containers plus filtered rows from the filtered containers - SimpleFilter.FilterClause getDomainDataFilterClause(Set copyContainers, Set filteredContainers, List domainFilters, TableInfo sourceTable, Set selectColumnNames); + FilterClause getDomainDataFilterClause(Set copyContainers, Set filteredContainers, List domainFilters, TableInfo sourceTable, Set selectColumnNames); - void addDomainDataFilterClause(SimpleFilter.OrClause orClause, DatabaseMigrationService.DataFilter filter, TableInfo sourceTable, Set selectColumnNames); + void addDomainDataFilterClause(OrClause orClause, DataFilter filter, TableInfo sourceTable, Set selectColumnNames); // Do any necessary clean up after the target table has been populated. notCopiedFilter selects all rows in the // source table that were NOT copied to the target table. (For example, rows in a global table not copied due to @@ -49,8 +52,7 @@ public interface MigrationSchemaHandler void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema); - // TODO: Return Collection, indicating which attachment types it handled? - void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema); + void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema, Set copyContainers); @NotNull Collection getAttachmentTypes(); diff --git a/assay/src/org/labkey/assay/AssayResultMigrationSchemaHandler.java b/assay/src/org/labkey/assay/AssayResultMigrationSchemaHandler.java index 7acaa367818..bf1499fb42a 100644 --- a/assay/src/org/labkey/assay/AssayResultMigrationSchemaHandler.java +++ b/assay/src/org/labkey/assay/AssayResultMigrationSchemaHandler.java @@ -12,14 +12,19 @@ import org.labkey.api.data.SimpleFilter.SQLClause; import org.labkey.api.data.SqlSelector; import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.migration.AssaySkipContainers; import org.labkey.api.migration.DatabaseMigrationService.DataFilter; import org.labkey.api.migration.DefaultMigrationSchemaHandler; import org.labkey.api.migration.ExperimentDeleteService; import org.labkey.api.util.GUID; +import org.labkey.api.util.StringUtilsLabKey; import org.labkey.api.util.logging.LogHelper; import org.labkey.assay.plate.PlateReplicateStatsDomainKind; import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; import java.util.Set; class AssayResultMigrationSchemaHandler extends DefaultMigrationSchemaHandler @@ -40,18 +45,30 @@ private boolean skipTable(TableInfo sourceTable) @Override public FilterClause getContainerClause(TableInfo sourceTable, Set containers) { - return new SQLClause(skipTable(sourceTable) ? - new SQLFragment("1 = 0") : - new SQLFragment("DataId IN (SELECT RowId FROM exp.Data WHERE Container") - .appendInClause(containers, sourceTable.getSqlDialect()) - .append(")") - ); + final SQLFragment sql; + + if (skipTable(sourceTable)) + { + sql = new SQLFragment("1 = 0"); + } + else + { + Set containerIds = new HashSet<>(containers); + containerIds.removeAll(AssaySkipContainers.getContainers()); + sql = new SQLFragment("DataId IN (SELECT RowId FROM exp.Data WHERE Container") + .appendInClause(containerIds, sourceTable.getSqlDialect()) + .append(")"); + } + + return new SQLClause(sql); } @Override public void addDomainDataFilterClause(OrClause orClause, DataFilter filter, TableInfo sourceTable, Set selectColumnNames) { - // We want no rows from containers with a domain data filter, so don't add any clauses + // No filtering on assay results for now; just add the passed in containers. Note that these will be filtered + // if AssaySkipContainers is configured. + orClause.addClause(getContainerClause(sourceTable, filter.containers())); } @Override @@ -78,6 +95,9 @@ public void afterTable(TableInfo sourceTable, TableInfo targetTable, SimpleFilte // Delete exp.Data, exp.Object, etc. rows associated with the rows that weren't copied ExperimentDeleteService.get().deleteDataRows(notCopiedObjectIds); } + + // TODO: Temp! + LOG.info(" " + StringUtilsLabKey.pluralize(new TableSelector(sourceTable, Collections.singleton("DataId")).stream(Integer.class).distinct().count(), "distinct DataId")); } } } diff --git a/core/src/org/labkey/core/CoreMigrationSchemaHandler.java b/core/src/org/labkey/core/CoreMigrationSchemaHandler.java index 603bda092ec..2e42998c397 100644 --- a/core/src/org/labkey/core/CoreMigrationSchemaHandler.java +++ b/core/src/org/labkey/core/CoreMigrationSchemaHandler.java @@ -2,6 +2,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.labkey.api.attachments.AttachmentCache; import org.labkey.api.attachments.AttachmentType; import org.labkey.api.attachments.LookAndFeelResourceType; import org.labkey.api.data.CompareType; @@ -192,14 +193,19 @@ public void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema s } @Override - public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema) + public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema, Set copyContainers) { // Default handling for core's standard attachment types - super.copyAttachments(configuration, sourceSchema, targetSchema); + super.copyAttachments(configuration, sourceSchema, targetSchema, copyContainers); // Special handling for LookAndFeelResourceType, which must select from the source database - SQLFragment sql = new SQLFragment(); - LookAndFeelResourceType.get().addWhereSql(sql, "Parent", "DocumentName"); + SQLFragment sql = new SQLFragment() + .append("Parent").appendInClause(copyContainers, sourceSchema.getSqlDialect()) + .append("AND (DocumentName IN (?, ?) OR ") + .add(AttachmentCache.FAVICON_FILE_NAME) + .add(AttachmentCache.STYLESHEET_FILE_NAME) + .append("DocumentName LIKE '" + AttachmentCache.LOGO_FILE_NAME_PREFIX + "%' OR ") + .append("DocumentName LIKE '" + AttachmentCache.MOBILE_LOGO_FILE_NAME_PREFIX + "%')"); copyAttachments(configuration, sourceSchema, new SQLClause(sql), LookAndFeelResourceType.get()); } diff --git a/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java b/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java index 7bc86ef411c..47b83be387d 100644 --- a/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java +++ b/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java @@ -18,6 +18,7 @@ import org.labkey.api.exp.OntologyManager; import org.labkey.api.exp.api.ExpProtocolAttachmentType; import org.labkey.api.exp.api.ExpRunAttachmentType; +import org.labkey.api.migration.AssaySkipContainers; import org.labkey.api.migration.DatabaseMigrationConfiguration; import org.labkey.api.migration.DefaultMigrationSchemaHandler; import org.labkey.api.query.FieldKey; @@ -26,6 +27,7 @@ import org.labkey.experiment.api.ExperimentServiceImpl; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Set; @@ -75,8 +77,17 @@ public List getTablesToCopy() @Override public FilterClause getContainerClause(TableInfo sourceTable, Set containers) { +// Set assayFilteredContainers = assayFilteredContainers(containers); return switch (sourceTable.getName()) { +// case "ExperimentRun", "ProtocolApplication" -> super.getContainerClause(sourceTable, assayFilteredContainers); +// case "Data" -> new AndClause( +// new InClause(FieldKey.fromParts("Container"), containers), +// new OrClause( +// new CompareClause(FieldKey.fromParts("RunId"), CompareType.ISBLANK, null), +// new InClause(FieldKey.fromParts("RunId", "Container"), assayFilteredContainers) +// ) +// ); case "DataInput" -> new AndClause( new InClause(FieldKey.fromParts("DataId", "Container"), containers), new InClause(FieldKey.fromParts("TargetApplicationId", "RunId", "Container"), containers) @@ -125,6 +136,13 @@ public FilterClause getContainerClause(TableInfo sourceTable, Set containe }; } + private Set assayFilteredContainers(Set containers) + { + Set filteredContainers = new HashSet<>(containers); + filteredContainers.removeAll(AssaySkipContainers.getContainers()); + return filteredContainers; + } + @Override public void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema) { diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index d37893fba01..67ab8161805 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -72,7 +72,6 @@ import org.labkey.api.exp.xar.LsidUtils; import org.labkey.api.files.FileContentService; import org.labkey.api.files.TableUpdaterFileListener; -import org.labkey.api.migration.AssaySkipFilter; import org.labkey.api.migration.DatabaseMigrationService; import org.labkey.api.migration.ExperimentDeleteService; import org.labkey.api.module.ModuleContext; @@ -880,7 +879,6 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM DataClassMigrationSchemaHandler dcHandler = new DataClassMigrationSchemaHandler(); DatabaseMigrationService.get().registerSchemaHandler(dcHandler); ExperimentDeleteService.setInstance(dcHandler); - DatabaseMigrationService.get().registerMigrationFilter(new AssaySkipFilter()); } @Override From 0cefc57ad269cbc8eae75fd1fa8e9dbd75bd0951 Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Tue, 11 Nov 2025 15:45:53 -0800 Subject: [PATCH 12/12] Filter assaywell and assay schema based on AssaySkipContainers --- .../api/migration/AssaySkipContainers.java | 8 +++++++ assay/src/org/labkey/assay/AssayModule.java | 22 ++++++++++++++++--- .../AssayResultMigrationSchemaHandler.java | 5 +---- .../ExperimentMigrationSchemaHandler.java | 9 -------- 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/api/src/org/labkey/api/migration/AssaySkipContainers.java b/api/src/org/labkey/api/migration/AssaySkipContainers.java index ca688510ab7..8e38eb20a75 100644 --- a/api/src/org/labkey/api/migration/AssaySkipContainers.java +++ b/api/src/org/labkey/api/migration/AssaySkipContainers.java @@ -2,6 +2,7 @@ import org.labkey.api.util.GUID; +import java.util.HashSet; import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; @@ -23,4 +24,11 @@ public static Set getContainers() { return SKIP_CONTAINERS; } + + public static Set getFilteredContainers(Set containers) + { + Set filteredContainers = new HashSet<>(containers); + filteredContainers.removeAll(AssaySkipContainers.getContainers()); + return filteredContainers; + } } diff --git a/assay/src/org/labkey/assay/AssayModule.java b/assay/src/org/labkey/assay/AssayModule.java index df7bf953181..fc095cdbbc3 100644 --- a/assay/src/org/labkey/assay/AssayModule.java +++ b/assay/src/org/labkey/assay/AssayModule.java @@ -39,6 +39,9 @@ import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.ContainerType; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter.FilterClause; +import org.labkey.api.data.SimpleFilter.SQLClause; import org.labkey.api.data.TableInfo; import org.labkey.api.data.UpgradeCode; import org.labkey.api.data.generator.DataGeneratorRegistry; @@ -46,6 +49,7 @@ import org.labkey.api.exp.api.ExpProtocol; import org.labkey.api.exp.api.ExperimentService; import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.migration.AssaySkipContainers; import org.labkey.api.migration.DatabaseMigrationService; import org.labkey.api.migration.DefaultMigrationSchemaHandler; import org.labkey.api.module.AdminLinkManager; @@ -66,6 +70,7 @@ import org.labkey.api.security.roles.RoleManager; import org.labkey.api.usageMetrics.UsageMetricsService; import org.labkey.api.util.ContextListener; +import org.labkey.api.util.GUID; import org.labkey.api.util.JspTestCase; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.StartupListener; @@ -296,15 +301,26 @@ public void moduleStartupComplete(ServletContext servletContext) { return PlateTypeTable.NAME.equals(sourceTable.getName()) ? SITE_WIDE_TABLE : super.getContainerFieldKey(sourceTable); } + + @Override + // Override to filter the container set + public FilterClause getContainerClause(TableInfo sourceTable, Set containers) + { + return super.getContainerClause(sourceTable, AssaySkipContainers.getFilteredContainers(containers)); + } }); - // Tables in the "assaywell" provisioned schema are all single-container, so no filtering is needed + // Tables in the "assaywell" provisioned schema join to assay.Well to find their container DatabaseMigrationService.get().registerSchemaHandler(new DefaultMigrationSchemaHandler(PlateMetadataDomainKind.getSchema()) { @Override - public @Nullable FieldKey getContainerFieldKey(TableInfo sourceTable) + public FilterClause getContainerClause(TableInfo sourceTable, Set containers) { - return SITE_WIDE_TABLE; + return new SQLClause( + new SQLFragment("LSID IN (SELECT LSID FROM assay.Well WHERE Container") + .appendInClause(AssaySkipContainers.getFilteredContainers(containers), sourceTable.getSqlDialect()) + .append(")") + ); } }); diff --git a/assay/src/org/labkey/assay/AssayResultMigrationSchemaHandler.java b/assay/src/org/labkey/assay/AssayResultMigrationSchemaHandler.java index bf1499fb42a..16e0a57ad82 100644 --- a/assay/src/org/labkey/assay/AssayResultMigrationSchemaHandler.java +++ b/assay/src/org/labkey/assay/AssayResultMigrationSchemaHandler.java @@ -24,7 +24,6 @@ import java.util.Collection; import java.util.Collections; -import java.util.HashSet; import java.util.Set; class AssayResultMigrationSchemaHandler extends DefaultMigrationSchemaHandler @@ -53,10 +52,8 @@ public FilterClause getContainerClause(TableInfo sourceTable, Set containe } else { - Set containerIds = new HashSet<>(containers); - containerIds.removeAll(AssaySkipContainers.getContainers()); sql = new SQLFragment("DataId IN (SELECT RowId FROM exp.Data WHERE Container") - .appendInClause(containerIds, sourceTable.getSqlDialect()) + .appendInClause(AssaySkipContainers.getFilteredContainers(containers), sourceTable.getSqlDialect()) .append(")"); } diff --git a/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java b/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java index 47b83be387d..db5b16f572d 100644 --- a/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java +++ b/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java @@ -18,7 +18,6 @@ import org.labkey.api.exp.OntologyManager; import org.labkey.api.exp.api.ExpProtocolAttachmentType; import org.labkey.api.exp.api.ExpRunAttachmentType; -import org.labkey.api.migration.AssaySkipContainers; import org.labkey.api.migration.DatabaseMigrationConfiguration; import org.labkey.api.migration.DefaultMigrationSchemaHandler; import org.labkey.api.query.FieldKey; @@ -27,7 +26,6 @@ import org.labkey.experiment.api.ExperimentServiceImpl; import java.util.Collection; -import java.util.HashSet; import java.util.List; import java.util.Set; @@ -136,13 +134,6 @@ public FilterClause getContainerClause(TableInfo sourceTable, Set containe }; } - private Set assayFilteredContainers(Set containers) - { - Set filteredContainers = new HashSet<>(containers); - filteredContainers.removeAll(AssaySkipContainers.getContainers()); - return filteredContainers; - } - @Override public void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, DbSchema targetSchema) {