Write file paths, etc.

api/src/org/labkey/api/exp/list/ListImportProgress.java → api/src/org/labkey/api/dataiterator/ImportProgress.java

@@ -1,27 +1,22 @@
-/*
- * Copyright (c) 2009-2014 LabKey Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.labkey.api.exp.list;
-
-/*
-* User: adam
-* Date: Dec 23, 2009
-* Time: 3:05:51 PM
-*/
-public interface ListImportProgress
-{
-    void setTotalRows(int rows);
-    void setCurrentRow(int currentRow);
-}
+/*
+ * Copyright (c) 2009-2014 LabKey Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.labkey.api.dataiterator;
+
+public interface ImportProgress
+{
+    void setTotalRows(int rows);
+    void setCurrentRow(int currentRow);
+}

api/src/org/labkey/api/dataiterator/Pump.java

@@ -17,7 +17,6 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.labkey.api.exp.list.ListImportProgress;
 import org.labkey.api.query.BatchValidationException;
 
 import java.io.IOException;
@@ -34,7 +33,7 @@ public class Pump implements Runnable
     final BatchValidationException _errors;
     int _errorLimit = Integer.MAX_VALUE;
     long _rowCount = 0;
-    ListImportProgress _progress = null;
+    ImportProgress _progress = null;
 
     public Pump(DataIterator it, DataIteratorContext context)
     {
@@ -50,7 +49,7 @@ public Pump(DataIteratorBuilder builder,  DataIteratorContext context)
         _errors = context.getErrors();
     }
 
-    public void setProgress(ListImportProgress progress)
+    public void setProgress(ImportProgress progress)
     {
         _progress = progress;
     }

api/src/org/labkey/api/exp/list/ListDefinition.java

@@ -23,6 +23,7 @@
 import org.labkey.api.data.ContainerFilter;
 import org.labkey.api.data.LookupResolutionType;
 import org.labkey.api.data.TableInfo;
+import org.labkey.api.dataiterator.ImportProgress;
 import org.labkey.api.exp.DomainNotFoundException;
 import org.labkey.api.exp.PropertyType;
 import org.labkey.api.exp.property.Domain;
@@ -269,8 +270,8 @@ public static BodySetting getForValue(int value)
     ListItem getListItemForEntityId(String entityId, User user);
 
     int insertListItems(User user, Container container, List<ListItem> listItems) throws IOException;
-    int insertListItems(User user, Container container, DataLoader loader, @NotNull BatchValidationException errors, @Nullable VirtualFile attachmentDir, @Nullable ListImportProgress progress, boolean supportAutoIncrementKey, LookupResolutionType lookupResolutionType) throws IOException;
-    int importListItems(User user, Container container, DataLoader loader, @NotNull BatchValidationException errors, @Nullable VirtualFile attachmentDir, @Nullable ListImportProgress progress, boolean supportAutoIncrementKey, LookupResolutionType lookupResolutionType, QueryUpdateService.InsertOption insertOption) throws IOException;
+    int insertListItems(User user, Container container, DataLoader loader, @NotNull BatchValidationException errors, @Nullable VirtualFile attachmentDir, @Nullable ImportProgress progress, boolean supportAutoIncrementKey, LookupResolutionType lookupResolutionType) throws IOException;
+    int importListItems(User user, Container container, DataLoader loader, @NotNull BatchValidationException errors, @Nullable VirtualFile attachmentDir, @Nullable ImportProgress progress, boolean supportAutoIncrementKey, LookupResolutionType lookupResolutionType, QueryUpdateService.InsertOption insertOption) throws IOException;
 
     @Nullable TableInfo getTable(User user);
     @Nullable TableInfo getTable(User user, Container c);

api/src/org/labkey/api/migration/DatabaseMigrationConfiguration.java

@@ -7,7 +7,10 @@
 import org.labkey.api.data.DbScope;
 import org.labkey.api.data.TableInfo;
 import org.labkey.api.data.TableSelector;
+import org.labkey.api.util.GUID;
+import org.labkey.api.util.Pair;
 
+import java.io.PrintWriter;
 import java.util.Set;
 import java.util.function.Predicate;
 
@@ -21,5 +24,8 @@ default void beforeMigration(){}
     Predicate<String> getColumnNameFilter();
     @Nullable TableSelector getTableSelector(DbSchemaType schemaType, TableInfo sourceTable, TableInfo targetTable, Set<String> selectColumnNames, MigrationSchemaHandler schemaHandler, @Nullable MigrationTableHandler tableHandler);
     default void copyAttachments(DbSchema sourceSchema, DbSchema targetSchema, MigrationSchemaHandler schemaHandler){}
-    default void afterMigration(){}
+    default @Nullable Pair<FilePathWriter, Set<GUID>> initializeFilePathWriter()
+    {
+        return null;
+    }
 }

api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java

@@ -306,15 +306,15 @@ public static void afterMigration() throws InterruptedException
         if (unseen.isEmpty())
             LOG.info("All AttachmentTypes have been seen");
         else
-            throw new ConfigurationException("These AttachmentTypes have not been seen: " + unseen.stream().map(type -> type.getClass().getSimpleName()).collect(Collectors.joining(", ")));
+            LOG.error("These AttachmentTypes have not been seen: {}", unseen.stream().map(type -> type.getClass().getSimpleName()).collect(Collectors.joining(", ")));
 
         // Shut down the attachment JobRunner
         LOG.info("Waiting for attachments background transfer to complete");
         ATTACHMENT_JOB_RUNNER.shutdown();
-        if (ATTACHMENT_JOB_RUNNER.awaitTermination(1, TimeUnit.HOURS))
+        if (ATTACHMENT_JOB_RUNNER.awaitTermination(2, TimeUnit.HOURS))
             LOG.info("Attachments background transfer is complete");
         else
-            LOG.error("Attachments background transfer did not complete after one hour! Giving up.");
+            LOG.error("Attachments background transfer did not complete after two hours! Giving up.");
     }
 
     @Override
@@ -332,4 +332,9 @@ public void afterSchema(DatabaseMigrationConfiguration configuration, DbSchema s
     public void afterMigration(DatabaseMigrationConfiguration configuration)
     {
     }
+
+    @Override
+    public void writeFilePaths(FilePathWriter writer, Set<GUID> guids)
+    {
+    }
 }

api/src/org/labkey/api/migration/FilePathWriter.java

@@ -0,0 +1,42 @@
+package org.labkey.api.migration;
+
+import org.labkey.api.data.ContainerManager;
+import org.labkey.api.files.FileContentService;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.PrintWriter;
+import java.nio.file.Path;
+
+public class FilePathWriter implements Closeable
+{
+    private final PrintWriter _out;
+    private final Path _rootPath;
+
+    public FilePathWriter(PrintWriter out)
+    {
+        _out = out;
+        _rootPath = FileContentService.get().getFileRoot(ContainerManager.getRoot()).toPath();
+    }
+
+    public void write(File file)
+    {
+        _out.println(_rootPath.relativize(file.toPath()).normalize());
+    }
+
+    public void println(String s)
+    {
+        _out.println(s);
+    }
+
+    public void println()
+    {
+        _out.println();
+    }
+
+    @Override
+    public void close()
+    {
+        _out.close();
+    }
+}

api/src/org/labkey/api/migration/MigrationSchemaHandler.java

@@ -12,6 +12,7 @@
 import org.labkey.api.query.FieldKey;
 import org.labkey.api.util.GUID;
 
+import java.io.PrintWriter;
 import java.util.Collection;
 import java.util.List;
 import java.util.Set;
@@ -57,4 +58,6 @@ public interface MigrationSchemaHandler
     @NotNull Collection<AttachmentType> getAttachmentTypes();
 
     void afterMigration(DatabaseMigrationConfiguration configuration);
+
+    void writeFilePaths(FilePathWriter writer, Set<GUID> guids);
 }

api/src/org/labkey/api/module/ModuleLoader.java

@@ -678,9 +678,6 @@ private void doInit(Execution execution) throws ServletException
         _log.info("Server installation GUID: {}, server session GUID: {}", AppProps.getInstance().getServerGUID(), AppProps.getInstance().getServerSessionGUID());
         _log.info("Deploying to context path {}", AppProps.getInstance().getContextPath());
 
-        // Temporary logging to help track down issues we're having with upgrading XMLBeans from v5.2.0. TODO: Remove after upgrade
-        _log.info("XMLBeans version: {}", XmlBeans.getVersion());
-
         synchronized (_modulesLock)
         {
             checkForRenamedModules();

experiment/src/org/labkey/experiment/DataClassMigrationSchemaHandler.java

@@ -24,6 +24,7 @@
 import org.labkey.api.migration.DatabaseMigrationService.DataFilter;
 import org.labkey.api.migration.DefaultMigrationSchemaHandler;
 import org.labkey.api.migration.ExperimentDeleteService;
+import org.labkey.api.migration.FilePathWriter;
 import org.labkey.api.query.FieldKey;
 import org.labkey.api.util.GUID;
 import org.labkey.api.util.StringUtilsLabKey;
@@ -95,8 +96,8 @@ public void afterTable(TableInfo sourceTable, TableInfo targetTable, SimpleFilte
 
         // Select all ObjectIds associated with the not-copied rows from the source database
         SQLFragment objectIdSql = new SQLFragment("SELECT ObjectId FROM exp.Data d INNER JOIN ")
-                .appendIdentifier(sourceTable.getSelectName())
-                .append(" dc ON d.LSID = dc.LSID");
+            .appendIdentifier(sourceTable.getSelectName())
+            .append(" dc ON d.LSID = dc.LSID");
 
         // Don't create an empty IN clause; need to work around issue where "NOT xxx IN (NULL)" evaluates to NULL.
         if (!copiedLsids.isEmpty())
@@ -134,7 +135,7 @@ public void afterTable(TableInfo sourceTable, TableInfo targetTable, SimpleFilte
                     }
                 })
                 .forEach(SEQUENCE_IDS::add);
-            LOG.info("{} added to the SequenceIdentity set",
+            LOG.info("   {} added to the SequenceIdentity set",
                 StringUtilsLabKey.pluralize(SEQUENCE_IDS.size() - startSize, "unique SequenceId was", "unique SequenceIds were"));
         }
     }
@@ -202,4 +203,11 @@ public FilterClause getTableFilterClause(TableInfo sourceTable, Set<GUID> contai
     {
         return List.of(ExpDataClassType.get());
     }
+
+    @Override
+    public void writeFilePaths(FilePathWriter writer, Set<GUID> guids)
+    {
+        // TODO: Enumerate FileLink fields in data classes in the filtered containers (guids) and write out those file
+        // paths. Current client has none.
+    }
 }

experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java

@@ -261,6 +261,15 @@ public FilterClause getContainerClause(TableInfo sourceTable, Set<GUID> containe
                     .appendInClause(containers, sourceTable.getSqlDialect())
                     .append(")")
             );
+            case "MaterialAliasMap" -> new AndClause(
+                new InClause(FieldKey.fromParts("Container"), containers),
+                // The below effectively matches the "Material" conditions above, since MaterialAliasMap has an FK to Material
+                new InClause(FieldKey.fromParts("LSID", "Container"), containers),
+                new OrClause(
+                    new CompareClause(FieldKey.fromParts("LSID", "RunId"), CompareType.ISBLANK, null),
+                    new InClause(FieldKey.fromParts("LSID", "RunId", "Container"), containers)
+                )
+            );
             case "ObjectLegacyNames" -> new SQLClause(
                 new SQLFragment("ObjectId IN (SELECT ObjectId FROM exp.Object WHERE Container")
                     .appendInClause(containers, sourceTable.getSqlDialect())

experiment/src/org/labkey/experiment/SampleTypeMigrationSchemaHandler.java

@@ -14,6 +14,7 @@
 import org.labkey.api.exp.api.SampleTypeDomainKind;
 import org.labkey.api.migration.DatabaseMigrationService.DataFilter;
 import org.labkey.api.migration.DefaultMigrationSchemaHandler;
+import org.labkey.api.migration.FilePathWriter;
 import org.labkey.api.util.GUID;
 import org.labkey.api.util.logging.LogHelper;
 
@@ -168,4 +169,11 @@ public void afterTable(TableInfo sourceTable, TableInfo targetTable, SimpleFilte
             ExperimentMigrationSchemaHandler.deleteObjectIds(objectIdClause);
         }
     }
+
+    @Override
+    public void writeFilePaths(FilePathWriter writer, Set<GUID> guids)
+    {
+        // TODO: Enumerate FileLink fields in sample types in the filtered containers (guids) and write out those file
+        // paths. Current client has none.
+    }
 }

list/src/org/labkey/list/model/ListDefinitionImpl.java

@@ -30,13 +30,13 @@
 import org.labkey.api.data.SimpleFilter;
 import org.labkey.api.data.TableInfo;
 import org.labkey.api.data.TableSelector;
+import org.labkey.api.dataiterator.ImportProgress;
 import org.labkey.api.exp.DomainNotFoundException;
 import org.labkey.api.exp.Lsid;
 import org.labkey.api.exp.ObjectProperty;
 import org.labkey.api.exp.TemplateInfo;
 import org.labkey.api.exp.api.ExperimentService;
 import org.labkey.api.exp.list.ListDefinition;
-import org.labkey.api.exp.list.ListImportProgress;
 import org.labkey.api.exp.list.ListItem;
 import org.labkey.api.exp.property.Domain;
 import org.labkey.api.exp.property.DomainProperty;
@@ -610,13 +610,13 @@ public int insertListItems(User user, Container container, List<ListItem> listIt
 
 
     @Override
-    public int insertListItems(User user, Container container, DataLoader loader, @NotNull BatchValidationException errors, @Nullable VirtualFile attachmentDir, @Nullable ListImportProgress progress, boolean supportAutoIncrementKey, LookupResolutionType lookupResolutionType)
+    public int insertListItems(User user, Container container, DataLoader loader, @NotNull BatchValidationException errors, @Nullable VirtualFile attachmentDir, @Nullable ImportProgress progress, boolean supportAutoIncrementKey, LookupResolutionType lookupResolutionType)
     {
         return importListItems(user, container, loader, errors, attachmentDir, progress, supportAutoIncrementKey, lookupResolutionType, QueryUpdateService.InsertOption.INSERT);
     }
 
     @Override
-    public int importListItems(User user, Container container, DataLoader loader, @NotNull BatchValidationException errors, @Nullable VirtualFile attachmentDir, @Nullable ListImportProgress progress, boolean supportAutoIncrementKey, LookupResolutionType lookupResolutionType, QueryUpdateService.InsertOption insertOption)
+    public int importListItems(User user, Container container, DataLoader loader, @NotNull BatchValidationException errors, @Nullable VirtualFile attachmentDir, @Nullable ImportProgress progress, boolean supportAutoIncrementKey, LookupResolutionType lookupResolutionType, QueryUpdateService.InsertOption insertOption)
     {
         ListQuerySchema schema = new ListQuerySchema(user, container);
         TableInfo table = schema.getTable(_def.getName());

list/src/org/labkey/list/model/ListQueryUpdateService.java

@@ -42,11 +42,11 @@
 import org.labkey.api.dataiterator.DataIteratorBuilder;
 import org.labkey.api.dataiterator.DataIteratorContext;
 import org.labkey.api.dataiterator.DetailedAuditLogDataIterator;
+import org.labkey.api.dataiterator.ImportProgress;
 import org.labkey.api.exp.ObjectProperty;
 import org.labkey.api.exp.PropertyType;
 import org.labkey.api.exp.api.ExperimentService;
 import org.labkey.api.exp.list.ListDefinition;
-import org.labkey.api.exp.list.ListImportProgress;
 import org.labkey.api.exp.list.ListItem;
 import org.labkey.api.exp.list.ListService;
 import org.labkey.api.exp.property.Domain;
@@ -229,7 +229,7 @@ private User getListUser(User user, Container container)
     }
 
     public int insertUsingDataIterator(DataLoader loader, User user, Container container, BatchValidationException errors, @Nullable VirtualFile attachmentDir,
-                                       @Nullable ListImportProgress progress, boolean supportAutoIncrementKey, InsertOption insertOption, LookupResolutionType lookupResolutionType)
+                                       @Nullable ImportProgress progress, boolean supportAutoIncrementKey, InsertOption insertOption, LookupResolutionType lookupResolutionType)
     {
         if (!_list.isVisible(user))
             throw new UnauthorizedException("You do not have permission to insert data into this table.");

search/src/org/labkey/search/SearchModule.java

@@ -29,6 +29,7 @@
 import org.labkey.api.data.UpgradeCode;
 import org.labkey.api.mbean.LabKeyManagement;
 import org.labkey.api.mbean.SearchMXBean;
+import org.labkey.api.migration.DatabaseMigrationConfiguration;
 import org.labkey.api.migration.DatabaseMigrationService;
 import org.labkey.api.migration.DefaultMigrationSchemaHandler;
 import org.labkey.api.module.DefaultModule;
@@ -195,6 +196,13 @@ public List<TableInfo> getTablesToCopy()
             {
                 return List.of(); // Leave empty -- target server will re-index all documents
             }
+
+            @Override
+            public void afterMigration(DatabaseMigrationConfiguration configuration)
+            {
+                // Clear index and all last indexed tracking
+                SearchService.get().deleteIndex("Database was just migrated");
+            }
         });
     }
 

specimen/src/org/labkey/specimen/importer/SpecimenImporter.java

@@ -57,6 +57,7 @@
 import org.labkey.api.dataiterator.DataIteratorBuilder;
 import org.labkey.api.dataiterator.DataIteratorContext;
 import org.labkey.api.dataiterator.DataIteratorUtil;
+import org.labkey.api.dataiterator.ImportProgress;
 import org.labkey.api.dataiterator.ListofMapsDataIterator;
 import org.labkey.api.dataiterator.LoggingDataIterator;
 import org.labkey.api.dataiterator.MapDataIterator;
@@ -69,7 +70,6 @@
 import org.labkey.api.exp.api.ExpSampleType;
 import org.labkey.api.exp.api.ExperimentService;
 import org.labkey.api.exp.api.SampleTypeService;
-import org.labkey.api.exp.list.ListImportProgress;
 import org.labkey.api.iterator.MarkableIterator;
 import org.labkey.api.pipeline.PipelineJob;
 import org.labkey.api.query.DefaultSchema;
@@ -1993,7 +1993,7 @@ public Object getValue(Map<String, Object> row)
         DataIteratorBuilder standardEtl = StandardDataIteratorBuilder.forInsert(target, specimenWrapped, getContainer(), getUser(), dix);
         DataIteratorBuilder persist = ((UpdateableTableInfo)target).persistRows(standardEtl, dix);
         Pump pump = new Pump(persist, dix);
-        pump.setProgress(new ListImportProgress()
+        pump.setProgress(new ImportProgress()
         {
             long heartBeat = HeartBeat.currentTimeMillis();