Reimplement folder export and import for workflow

api/schemas/expTypes.xsd

@@ -96,7 +96,7 @@
 			<xs:element name="Comments" type="string" minOccurs="0"/>
             <xs:element name="ExperimentLSID" type="string" minOccurs="0" maxOccurs="1" />
 			<xs:element name="Properties" type="exp:PropertyCollectionType" minOccurs="0"/>
-            <xs:element name="WorkflowTaskLSID" type="string" minOccurs="0" maxOccurs="1" />
+            <xs:element name="WorkflowTaskId" type="string" minOccurs="0" maxOccurs="1" />
             <xs:element name="ReplacedByRunLSID" type="string" minOccurs="0">
 				<xs:annotation>
 					<xs:documentation>Reference to another run that defines an updated version of this data</xs:documentation>

api/schemas/folder.xsd

@@ -93,6 +93,7 @@
                 <xs:element name="moduleProperties" minOccurs="0" type="modulePropertiesType"/>
                 <xs:element name="xar" minOccurs="0" type="exportDirType" />
                 <xs:element name="inventory" minOccurs="0" type="exportDirType"/>
+                <xs:element name="workflow" minOccurs="0" type="exportDirType"/>
                 <xs:element name="fileBrowserConfig" minOccurs="0" maxOccurs="1">
                     <xs:complexType>
                         <xs:attribute name="file" type="xs:string">

api/src/org/labkey/api/admin/FolderArchiveDataTypes.java

@@ -51,4 +51,5 @@ private FolderArchiveDataTypes() {}
     public static final String DATA_CLASS_DATA = "Data Class Data";
     public static final String INVENTORY = "Inventory locations and items";
     public static final String VIEW_CATEGORIES = "Categories";
+    public static final String WORKFLOW = "Workflow jobs, templates, tasks, actions and entities";
 }

api/src/org/labkey/api/admin/FolderImportContext.java

@@ -51,6 +51,7 @@ public class FolderImportContext extends AbstractFolderContext
     private String _xarJobId;
 
     private final HashSet<String> _importedReports = new HashSet<>();
+    private Map<Long, Long> _assayRunWorkflowTaskMap = new HashMap<>();
 
     private static final String FOLDER_IMPORT_DB_SEQUENCE_PREFIX = "FolderImportJobCounter-";
 
@@ -180,6 +181,16 @@ public void addImportedReport(ReportDescriptor d)
         _importedReports.add(ReportUtil.getSerializedName(d));
     }
 
+    public Map<Long, Long> getAssayRunWorkflowTaskMap()
+    {
+        return _assayRunWorkflowTaskMap;
+    }
+
+    public void setAssayRunWorkflowTaskMap(Map<Long, Long> assayRunWorkflowTaskMap)
+    {
+        _assayRunWorkflowTaskMap = assayRunWorkflowTaskMap;
+    }
+
     @Override
     public AuditBehaviorType getAuditBehaviorType() throws Exception
     {

api/src/org/labkey/api/exp/api/ExpProtocol.java

@@ -165,21 +165,5 @@ default String getDocumentId()
         return String.join(":",getContainer().getId(), "assay", String.valueOf(getRowId()));
     }
 
-    static boolean isSampleWorkflowJobProtocol(String lsid)
-    {
-        return lsid.contains(ExperimentService.SAMPLE_MANAGEMENT_JOB_PROTOCOL_PREFIX);
-    }
-
-    static boolean isSampleWorkflowTaskProtocol(String lsid)
-    {
-        return lsid.contains(ExperimentService.SAMPLE_MANAGEMENT_TASK_PROTOCOL_PREFIX);
-    }
-
-    // TODO remove this and its relatives once the workflow folder import/export rewrite has happened.
-    static boolean isSampleWorkflowProtocol(String lsid)
-    {
-        return isSampleWorkflowTaskProtocol(lsid) || isSampleWorkflowJobProtocol(lsid);
-    }
-
     Map<String, Object> getAuditRecordMap(AssayProvider provider);
 }

api/src/org/labkey/api/exp/api/ExperimentService.java

@@ -393,6 +393,8 @@ ValidationException updateDataClass(
      */
     @NotNull List<? extends ExpMaterial> getExpMaterialsByName(String name, @Nullable Container container, User user);
 
+    @NotNull List<? extends ExpMaterial> getExpMaterialsByName(@NotNull Collection<String> names, @NotNull String sampleTypeName, @NotNull Container container, User user);
+
     @Nullable ExpData findExpData(
         Container c,
         User user,

experiment/src/org/labkey/experiment/XarExporter.java

@@ -330,15 +330,8 @@ public void addExperimentRun(ExpRun run) throws ExperimentException
             addProtocolApplication(application, run, xApplications);
         }
 
-        // TODO need to update this for run workflowTaskId support in XAR on workflow job folder export/import is supported
-        //ExpProtocolApplication workflowTask = run.getWorkflowTask();
-        //if (workflowTask != null)
-        //{
-        //    // Due to the way ProtocolApplication LSIDs are generated we can't actually round trip them the normal way
-        //    // via the LSIDRelativizer. Instead we construct an LSID out of the object ID with a custom prefix.
-        //    String workflowObjectId = Lsid.parse(workflowTask.getLSID()).getObjectId();
-        //    xRun.setWorkflowTaskLSID("${WorkflowTaskReference}:" + workflowObjectId);
-        //}
+        if (run.getWorkflowTaskId() != null)
+            xRun.setWorkflowTaskId(run.getWorkflowTaskId().toString());
 
         // get AssayService.get().getProvider(run).getXarCallbacks().beforeXarExportRun() with simple attempt at caching for common case
         if (null != AssayService.get())
@@ -1407,38 +1400,6 @@ private PropertyCollectionType getProperties(Map<String, ObjectProperty> propert
                             catch (URISyntaxException ignored) {}
                             simpleValue.setStringValue(link);
                         }
-                        // This property stores rowIds of assay designs; we need to translate them to LSIDs for export
-                        // TODO perhaps this property should hold protocol strings instead of rowIds
-                        else if (value.getPropertyURI().endsWith(":WorkflowTask#AssayTypes"))
-                        {
-                            String assayIdsString = value.getStringValue();
-                            if (!StringUtils.isEmpty(assayIdsString))
-                            {
-                                String[] assayIds = assayIdsString.split(",");
-                                List<String> protocolStrings = new ArrayList<>();
-                                List<ExpProtocol> protocols = AssayService.get().getAssayProtocols(value.getContainer());
-                                for (String assayId : assayIds)
-                                {
-                                    try
-                                    {
-                                        int assayRowId = Integer.parseInt(assayId);
-                                        Optional<ExpProtocol> protocol = protocols.stream().filter(p -> p.getRowId() == assayRowId).findFirst();
-                                        if (protocol.isPresent())
-                                            protocolStrings.add(relativizeLSIDPropertyValue(protocol.get().getLSID(), SimpleTypeNames.STRING));
-                                        else
-                                            logProgress("Unable to find protocol for assay id " + assayRowId + ".  Not included in values for " + value.getName() + ".");
-                                    }
-                                    catch (NumberFormatException ignore)
-                                    {
-                                        // assume it's an LSID and try to relativize it
-                                        protocolStrings.add(relativizeLSIDPropertyValue(assayId, SimpleTypeNames.STRING));
-                                    }
-                                }
-                                simpleValue.setStringValue(StringUtils.join(protocolStrings, ","));
-                            }
-                            else
-                                simpleValue.setStringValue(assayIdsString);
-                        }
                         else
                         {
                             simpleValue.setStringValue(relativizeLSIDPropertyValue(value.getStringValue(), SimpleTypeNames.STRING));

experiment/src/org/labkey/experiment/XarReader.java

@@ -29,6 +29,7 @@
 import org.fhcrc.cpas.exp.xml.*;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+import org.labkey.api.admin.FolderImportContext;
 import org.labkey.api.assay.AssayProvider;
 import org.labkey.api.assay.AssayService;
 import org.labkey.api.collections.LongHashMap;
@@ -138,7 +139,8 @@ public class XarReader extends AbstractXarImporter
 
     private final Set<String> _experimentLSIDs = new HashSet<>();
     private final Map<String, Integer> _propertyIdMap = new HashMap<>();
-    private final Map<Long, String> _runWorkflowTaskMap = new LongHashMap<>();
+    private final Map<Long, Long> _runWorkflowTaskMap = new LongHashMap<>();
+    private final FolderImportContext _folderImportContext;
     /** Retain replacement info so we can wire them up after all runs have been imported */
     private final Map<Long, String> _runReplacedByMap = new LongHashMap<>();
 
@@ -168,9 +170,15 @@ public class XarReader extends AbstractXarImporter
     private AuditBehaviorType _auditBehaviorType = null;
 
     public XarReader(XarSource source, PipelineJob job)
+    {
+        this(source, job, null);
+    }
+
+    public XarReader(XarSource source, PipelineJob job, FolderImportContext ctx)
     {
         super(source, job);
         _fileRootPath = getContainerFileRootPath(job.getContainer());
+        _folderImportContext = ctx;
     }
 
     public void setReloadExistingRuns(boolean reloadExistingRuns)
@@ -429,10 +437,8 @@ private void loadDoc() throws ExperimentException
                 }
             }
 
-            if (!_runWorkflowTaskMap.isEmpty())
-            {
-                saveRunWorkflowTaskIds();
-            }
+            if (_folderImportContext != null)
+                _folderImportContext.setAssayRunWorkflowTaskMap(_runWorkflowTaskMap);
 
             resolveReplacedByRunLSIDs();
 
@@ -1100,56 +1106,45 @@ private void loadExperimentRun(ExperimentRunType a, List<ExpMaterial> startingMa
             }
         }
 
-        if (run == null)
-        {
-            ExperimentRun vals = new ExperimentRun();
-            // todo not sure about having roots stored in database
-            // todo support substitutions here?
 
-            vals.setLSID(pRunLSID.toString());
+        ExperimentRun vals = new ExperimentRun();
+        // todo not sure about having roots stored in database
+        // todo support substitutions here?
 
-            vals.setName(trimString(a.getName()));
-            vals.setProtocolLSID(protocol.getLSID());
-            vals.setComments(trimString(a.getComments()));
+        vals.setLSID(pRunLSID.toString());
 
-            vals.setFilePathRoot(FileUtil.getAbsolutePath(_xarSource.getJobRootPath()));
-            vals.setContainer(getContainer());
-            // TODO need to update this for run workflowTaskId support in XAR on workflow job folder export/import is supported
-            //String workflowTaskLSID = a.getWorkflowTaskLSID();
-            //if (workflowTaskLSID != null)
-            //{
-            //    if (!workflowTaskLSID.startsWith("${WorkflowTaskReference}:"))
-            //        throw new XarFormatException("Invalid WorkflowTaskLSID encountered: " + workflowTaskLSID);
-            //
-            //    workflowTaskLSID = workflowTaskLSID.split(":")[1];
-            //}
-            if (_job != null)
-            {
-                // remember which job created the run so we can show this run on the job details page
-                vals.setJobId(PipelineService.get().getJobId(_job.getUser(), _job.getContainer(), _job.getJobGUID()));
-            }
+        vals.setName(trimString(a.getName()));
+        vals.setProtocolLSID(protocol.getLSID());
+        vals.setComments(trimString(a.getComments()));
 
-            ExpRunImpl impl = new ExpRunImpl(vals);
-            try
-            {
-                impl.save(getUser());
-                run = impl.getDataObject();
+        vals.setFilePathRoot(FileUtil.getAbsolutePath(_xarSource.getJobRootPath()));
+        vals.setContainer(getContainer());
 
-                //if (workflowTaskLSID != null)
-                //    _runWorkflowTaskMap.put(run.getRowId(), workflowTaskLSID);
+        if (_job != null)
+        {
+            // remember which job created the run so we can show this run on the job details page
+            vals.setJobId(PipelineService.get().getJobId(_job.getUser(), _job.getContainer(), _job.getJobGUID()));
+        }
 
-                String replacedByLSID = a.getReplacedByRunLSID();
-                if (replacedByLSID != null)
-                    // Save for later so that we can resolve after everything's been imported
-                {
-                    _runReplacedByMap.put(impl.getRowId(), replacedByLSID);
-                }
-            }
-            catch (BatchValidationException x)
+        ExpRunImpl impl = new ExpRunImpl(vals);
+        try
+        {
+            impl.save(getUser());
+            run = impl.getDataObject();
+            String taskIdStr = a.getWorkflowTaskId();
+            if (taskIdStr != null)
+                _runWorkflowTaskMap.put(run.getRowId(), Long.valueOf(taskIdStr));
+            String replacedByLSID = a.getReplacedByRunLSID();
+            if (replacedByLSID != null)
+                // Save for later so that we can resolve after everything's been imported
             {
-                throw new ExperimentException(x);
+                _runReplacedByMap.put(impl.getRowId(), replacedByLSID);
             }
         }
+        catch (BatchValidationException x)
+        {
+            throw new ExperimentException(x);
+        }
 
         if (experimentLSID != null)
         {
@@ -1208,46 +1203,6 @@ private void loadExperimentRun(ExperimentRunType a, List<ExpMaterial> startingMa
         getLog().debug("Finished loading ExperimentRun with LSID '" + runLSID + "'");
     }
 
-    /**
-     * // TODO need to update this for run workflowTaskId support in XAR
-     * This method runs last, and is used to wire up the Workflow Task FK relationship between Exp Runs and
-     * Exp ProtocolApplications. This needs to run last because we have no good way to guarantee import order and ensure
-     * all the appropriate ProtocolApplications are imported before the Exp Runs.
-     */
-    private void saveRunWorkflowTaskIds() throws ExperimentException
-    {
-        for (ExpRun run : _loadedRuns)
-        {
-            String objectId = _runWorkflowTaskMap.get(run.getRowId());
-
-            if (objectId != null)
-            {
-                List<? extends ExpProtocolApplication> protocolApplications = ExperimentService.get().getExpProtocolApplicationsByObjectId(getContainer(), objectId);
-
-                if (protocolApplications.size() > 1)
-                {
-                    throw new ExperimentException("Multiple ProtocolApplications found with object id: " + objectId);
-                }
-                else if (protocolApplications.isEmpty())
-                {
-                    getLog().warn("Could not find ProtocolApplication with LSID containing object id: " + objectId);
-                }
-                else
-                {
-                    run.setWorkflowTaskId(protocolApplications.get(0).getRowId());
-
-                    try {
-                        run.save(getUser());
-                    }
-                    catch (BatchValidationException e)
-                    {
-                        throw new ExperimentException(e);
-                    }
-                }
-            }
-        }
-    }
-
     public List<String> getProcessedRunsLSIDs()
     {
         return _processedRunsLSIDs;
@@ -1449,9 +1404,7 @@ private void loadProtocolApplication(ProtocolApplicationBaseType xmlProtocolApp,
         if ((protocolLSID.equals(SAMPLE_ALIQUOT_PROTOCOL_LSID) || protocolLSID.equals(SAMPLE_DERIVATION_PROTOCOL_LSID)) &&
                 !material.isOperationPermitted(SampleTypeService.SampleOperations.EditLineage))
             return SampleTypeService.get().getOperationNotPermittedMessage(Collections.singleton(material), SampleTypeService.SampleOperations.EditLineage);
-        if ((ExpProtocol.isSampleWorkflowProtocol(protocolLSID))
-                && !material.isOperationPermitted(SampleTypeService.SampleOperations.AddToWorkflow))
-            return SampleTypeService.get().getOperationNotPermittedMessage(Collections.singleton(material), SampleTypeService.SampleOperations.AddToWorkflow);
+
         return null;
     }
 

experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java

@@ -855,6 +855,23 @@ public List<ExpMaterialImpl> getExpMaterialsByName(@NotNull String name, @Nullab
             .toList();
     }
 
+    @NotNull
+    @Override
+    public List<ExpMaterialImpl> getExpMaterialsByName(@NotNull Collection<String> names, @NotNull String sampleTypeName, @NotNull Container container, User user)
+    {
+        ExpSampleType sampleType = SampleTypeService.get().getSampleType(container, user, sampleTypeName);
+        if (sampleType == null)
+            return Collections.emptyList();
+        SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(ExpMaterialTable.Column.Name.name()), names, IN);
+        filter.addCondition(FieldKey.fromParts("Container"), container);
+        filter.addCondition(FieldKey.fromParts("CpasType"), sampleType.getLSID());
+
+        return getExpMaterials(filter)
+                .stream()
+                .filter(m -> m.getContainer().hasPermission(user, ReadPermission.class) && m.getSampleType() != null)
+                .toList();
+    }
+
     public @Nullable ExpMaterialImpl getExpMaterial(SimpleFilter filter)
     {
         Material material = new TableSelector(getTinfoMaterial(), filter, null).getObject(Material.class);

experiment/src/org/labkey/experiment/samples/AbstractExpFolderImporter.java

@@ -157,7 +157,7 @@ else if (file.equalsIgnoreCase(XAR_RUNS_NAME) || file.equalsIgnoreCase(XAR_RUNS_
                             throw(e);
                         }
                         log.info("Importing the runs XAR file: " + runsXarFile.getName());
-                        XarReader runsReader = new FolderXarImporterFactory.FolderExportXarReader(runsXarSource, job);
+                        XarReader runsReader = new FolderXarImporterFactory.FolderExportXarReader(runsXarSource, job, ctx);
                         runsReader.setStrictValidateExistingSampleType(xarCtx.isStrictValidateExistingSampleType());
                         runsReader.parseAndLoad(false, ctx.getAuditBehaviorType());
                     }
@@ -220,7 +220,7 @@ protected XarReader getXarReader(@Nullable PipelineJob job, FolderImportContext
             log.error("Failed to initialize types XAR source", e);
             throw(e);
         }
-        return new FolderXarImporterFactory.FolderExportXarReader(typesXarSource, job);
+        return new FolderXarImporterFactory.FolderExportXarReader(typesXarSource, job, ctx);
     }
 
     protected PipelineJob getDummyPipelineJob(FolderImportContext ctx)

experiment/src/org/labkey/experiment/xar/FolderXarImporterFactory.java

@@ -141,7 +141,7 @@ protected XarSource createXarSource(FileLike file)
                     throw(e);
                 }
 
-                FolderExportXarReader reader = new FolderExportXarReader(xarSource, job);
+                FolderExportXarReader reader = new FolderExportXarReader(xarSource, job, ctx);
                 XarImportContext xarCtx = ctx.getContext(XarImportContext.class);
                 if (xarCtx != null)
                 {
@@ -238,6 +238,11 @@ public FolderExportXarReader(XarSource source, PipelineJob job)
             super(source, job);
         }
 
+        public FolderExportXarReader(XarSource source, PipelineJob job, FolderImportContext ctx)
+        {
+            super(source, job, ctx);
+        }
+
         @Override
         protected Container getContainer()
         {