diff --git a/api/schemas/expTypes.xsd b/api/schemas/expTypes.xsd index ba909bd0489..ef4ea5798ba 100644 --- a/api/schemas/expTypes.xsd +++ b/api/schemas/expTypes.xsd @@ -96,7 +96,7 @@ - + Reference to another run that defines an updated version of this data diff --git a/api/schemas/folder.xsd b/api/schemas/folder.xsd index e4a15cdeef7..ee473edc142 100644 --- a/api/schemas/folder.xsd +++ b/api/schemas/folder.xsd @@ -93,6 +93,7 @@ + diff --git a/api/src/org/labkey/api/admin/FolderArchiveDataTypes.java b/api/src/org/labkey/api/admin/FolderArchiveDataTypes.java index 2ebe7702407..67751f5cec8 100644 --- a/api/src/org/labkey/api/admin/FolderArchiveDataTypes.java +++ b/api/src/org/labkey/api/admin/FolderArchiveDataTypes.java @@ -51,4 +51,5 @@ private FolderArchiveDataTypes() {} public static final String DATA_CLASS_DATA = "Data Class Data"; public static final String INVENTORY = "Inventory locations and items"; public static final String VIEW_CATEGORIES = "Categories"; + public static final String WORKFLOW = "Workflow jobs, templates, tasks, actions and entities"; } diff --git a/api/src/org/labkey/api/admin/FolderImportContext.java b/api/src/org/labkey/api/admin/FolderImportContext.java index 786f5a31650..3a05a32f9df 100644 --- a/api/src/org/labkey/api/admin/FolderImportContext.java +++ b/api/src/org/labkey/api/admin/FolderImportContext.java @@ -51,6 +51,7 @@ public class FolderImportContext extends AbstractFolderContext private String _xarJobId; private final HashSet _importedReports = new HashSet<>(); + private Map _assayRunWorkflowTaskMap = new HashMap<>(); private static final String FOLDER_IMPORT_DB_SEQUENCE_PREFIX = "FolderImportJobCounter-"; @@ -180,6 +181,16 @@ public void addImportedReport(ReportDescriptor d) _importedReports.add(ReportUtil.getSerializedName(d)); } + public Map getAssayRunWorkflowTaskMap() + { + return _assayRunWorkflowTaskMap; + } + + public void setAssayRunWorkflowTaskMap(Map assayRunWorkflowTaskMap) + { + _assayRunWorkflowTaskMap = assayRunWorkflowTaskMap; + } + @Override public AuditBehaviorType getAuditBehaviorType() throws Exception { diff --git a/api/src/org/labkey/api/exp/api/ExpProtocol.java b/api/src/org/labkey/api/exp/api/ExpProtocol.java index 10593ad4d0b..2079789d1ac 100644 --- a/api/src/org/labkey/api/exp/api/ExpProtocol.java +++ b/api/src/org/labkey/api/exp/api/ExpProtocol.java @@ -165,21 +165,5 @@ default String getDocumentId() return String.join(":",getContainer().getId(), "assay", String.valueOf(getRowId())); } - static boolean isSampleWorkflowJobProtocol(String lsid) - { - return lsid.contains(ExperimentService.SAMPLE_MANAGEMENT_JOB_PROTOCOL_PREFIX); - } - - static boolean isSampleWorkflowTaskProtocol(String lsid) - { - return lsid.contains(ExperimentService.SAMPLE_MANAGEMENT_TASK_PROTOCOL_PREFIX); - } - - // TODO remove this and its relatives once the workflow folder import/export rewrite has happened. - static boolean isSampleWorkflowProtocol(String lsid) - { - return isSampleWorkflowTaskProtocol(lsid) || isSampleWorkflowJobProtocol(lsid); - } - Map getAuditRecordMap(AssayProvider provider); } diff --git a/api/src/org/labkey/api/exp/api/ExperimentService.java b/api/src/org/labkey/api/exp/api/ExperimentService.java index 2ce76f42943..3022b41482d 100644 --- a/api/src/org/labkey/api/exp/api/ExperimentService.java +++ b/api/src/org/labkey/api/exp/api/ExperimentService.java @@ -393,6 +393,8 @@ ValidationException updateDataClass( */ @NotNull List getExpMaterialsByName(String name, @Nullable Container container, User user); + @NotNull List getExpMaterialsByName(@NotNull Collection names, @NotNull String sampleTypeName, @NotNull Container container, User user); + @Nullable ExpData findExpData( Container c, User user, diff --git a/experiment/src/org/labkey/experiment/XarExporter.java b/experiment/src/org/labkey/experiment/XarExporter.java index f1e36d6af4f..1961b6016b8 100644 --- a/experiment/src/org/labkey/experiment/XarExporter.java +++ b/experiment/src/org/labkey/experiment/XarExporter.java @@ -330,15 +330,8 @@ public void addExperimentRun(ExpRun run) throws ExperimentException addProtocolApplication(application, run, xApplications); } - // TODO need to update this for run workflowTaskId support in XAR on workflow job folder export/import is supported - //ExpProtocolApplication workflowTask = run.getWorkflowTask(); - //if (workflowTask != null) - //{ - // // Due to the way ProtocolApplication LSIDs are generated we can't actually round trip them the normal way - // // via the LSIDRelativizer. Instead we construct an LSID out of the object ID with a custom prefix. - // String workflowObjectId = Lsid.parse(workflowTask.getLSID()).getObjectId(); - // xRun.setWorkflowTaskLSID("${WorkflowTaskReference}:" + workflowObjectId); - //} + if (run.getWorkflowTaskId() != null) + xRun.setWorkflowTaskId(run.getWorkflowTaskId().toString()); // get AssayService.get().getProvider(run).getXarCallbacks().beforeXarExportRun() with simple attempt at caching for common case if (null != AssayService.get()) @@ -1407,38 +1400,6 @@ private PropertyCollectionType getProperties(Map propert catch (URISyntaxException ignored) {} simpleValue.setStringValue(link); } - // This property stores rowIds of assay designs; we need to translate them to LSIDs for export - // TODO perhaps this property should hold protocol strings instead of rowIds - else if (value.getPropertyURI().endsWith(":WorkflowTask#AssayTypes")) - { - String assayIdsString = value.getStringValue(); - if (!StringUtils.isEmpty(assayIdsString)) - { - String[] assayIds = assayIdsString.split(","); - List protocolStrings = new ArrayList<>(); - List protocols = AssayService.get().getAssayProtocols(value.getContainer()); - for (String assayId : assayIds) - { - try - { - int assayRowId = Integer.parseInt(assayId); - Optional protocol = protocols.stream().filter(p -> p.getRowId() == assayRowId).findFirst(); - if (protocol.isPresent()) - protocolStrings.add(relativizeLSIDPropertyValue(protocol.get().getLSID(), SimpleTypeNames.STRING)); - else - logProgress("Unable to find protocol for assay id " + assayRowId + ". Not included in values for " + value.getName() + "."); - } - catch (NumberFormatException ignore) - { - // assume it's an LSID and try to relativize it - protocolStrings.add(relativizeLSIDPropertyValue(assayId, SimpleTypeNames.STRING)); - } - } - simpleValue.setStringValue(StringUtils.join(protocolStrings, ",")); - } - else - simpleValue.setStringValue(assayIdsString); - } else { simpleValue.setStringValue(relativizeLSIDPropertyValue(value.getStringValue(), SimpleTypeNames.STRING)); diff --git a/experiment/src/org/labkey/experiment/XarReader.java b/experiment/src/org/labkey/experiment/XarReader.java index fe4c3ac60b6..46a7480f427 100644 --- a/experiment/src/org/labkey/experiment/XarReader.java +++ b/experiment/src/org/labkey/experiment/XarReader.java @@ -29,6 +29,7 @@ import org.fhcrc.cpas.exp.xml.*; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.labkey.api.admin.FolderImportContext; import org.labkey.api.assay.AssayProvider; import org.labkey.api.assay.AssayService; import org.labkey.api.collections.LongHashMap; @@ -138,7 +139,8 @@ public class XarReader extends AbstractXarImporter private final Set _experimentLSIDs = new HashSet<>(); private final Map _propertyIdMap = new HashMap<>(); - private final Map _runWorkflowTaskMap = new LongHashMap<>(); + private final Map _runWorkflowTaskMap = new LongHashMap<>(); + private final FolderImportContext _folderImportContext; /** Retain replacement info so we can wire them up after all runs have been imported */ private final Map _runReplacedByMap = new LongHashMap<>(); @@ -168,9 +170,15 @@ public class XarReader extends AbstractXarImporter private AuditBehaviorType _auditBehaviorType = null; public XarReader(XarSource source, PipelineJob job) + { + this(source, job, null); + } + + public XarReader(XarSource source, PipelineJob job, FolderImportContext ctx) { super(source, job); _fileRootPath = getContainerFileRootPath(job.getContainer()); + _folderImportContext = ctx; } public void setReloadExistingRuns(boolean reloadExistingRuns) @@ -429,10 +437,8 @@ private void loadDoc() throws ExperimentException } } - if (!_runWorkflowTaskMap.isEmpty()) - { - saveRunWorkflowTaskIds(); - } + if (_folderImportContext != null) + _folderImportContext.setAssayRunWorkflowTaskMap(_runWorkflowTaskMap); resolveReplacedByRunLSIDs(); @@ -1100,56 +1106,45 @@ private void loadExperimentRun(ExperimentRunType a, List startingMa } } - if (run == null) - { - ExperimentRun vals = new ExperimentRun(); - // todo not sure about having roots stored in database - // todo support substitutions here? - vals.setLSID(pRunLSID.toString()); + ExperimentRun vals = new ExperimentRun(); + // todo not sure about having roots stored in database + // todo support substitutions here? - vals.setName(trimString(a.getName())); - vals.setProtocolLSID(protocol.getLSID()); - vals.setComments(trimString(a.getComments())); + vals.setLSID(pRunLSID.toString()); - vals.setFilePathRoot(FileUtil.getAbsolutePath(_xarSource.getJobRootPath())); - vals.setContainer(getContainer()); - // TODO need to update this for run workflowTaskId support in XAR on workflow job folder export/import is supported - //String workflowTaskLSID = a.getWorkflowTaskLSID(); - //if (workflowTaskLSID != null) - //{ - // if (!workflowTaskLSID.startsWith("${WorkflowTaskReference}:")) - // throw new XarFormatException("Invalid WorkflowTaskLSID encountered: " + workflowTaskLSID); - // - // workflowTaskLSID = workflowTaskLSID.split(":")[1]; - //} - if (_job != null) - { - // remember which job created the run so we can show this run on the job details page - vals.setJobId(PipelineService.get().getJobId(_job.getUser(), _job.getContainer(), _job.getJobGUID())); - } + vals.setName(trimString(a.getName())); + vals.setProtocolLSID(protocol.getLSID()); + vals.setComments(trimString(a.getComments())); - ExpRunImpl impl = new ExpRunImpl(vals); - try - { - impl.save(getUser()); - run = impl.getDataObject(); + vals.setFilePathRoot(FileUtil.getAbsolutePath(_xarSource.getJobRootPath())); + vals.setContainer(getContainer()); - //if (workflowTaskLSID != null) - // _runWorkflowTaskMap.put(run.getRowId(), workflowTaskLSID); + if (_job != null) + { + // remember which job created the run so we can show this run on the job details page + vals.setJobId(PipelineService.get().getJobId(_job.getUser(), _job.getContainer(), _job.getJobGUID())); + } - String replacedByLSID = a.getReplacedByRunLSID(); - if (replacedByLSID != null) - // Save for later so that we can resolve after everything's been imported - { - _runReplacedByMap.put(impl.getRowId(), replacedByLSID); - } - } - catch (BatchValidationException x) + ExpRunImpl impl = new ExpRunImpl(vals); + try + { + impl.save(getUser()); + run = impl.getDataObject(); + String taskIdStr = a.getWorkflowTaskId(); + if (taskIdStr != null) + _runWorkflowTaskMap.put(run.getRowId(), Long.valueOf(taskIdStr)); + String replacedByLSID = a.getReplacedByRunLSID(); + if (replacedByLSID != null) + // Save for later so that we can resolve after everything's been imported { - throw new ExperimentException(x); + _runReplacedByMap.put(impl.getRowId(), replacedByLSID); } } + catch (BatchValidationException x) + { + throw new ExperimentException(x); + } if (experimentLSID != null) { @@ -1208,46 +1203,6 @@ private void loadExperimentRun(ExperimentRunType a, List startingMa getLog().debug("Finished loading ExperimentRun with LSID '" + runLSID + "'"); } - /** - * // TODO need to update this for run workflowTaskId support in XAR - * This method runs last, and is used to wire up the Workflow Task FK relationship between Exp Runs and - * Exp ProtocolApplications. This needs to run last because we have no good way to guarantee import order and ensure - * all the appropriate ProtocolApplications are imported before the Exp Runs. - */ - private void saveRunWorkflowTaskIds() throws ExperimentException - { - for (ExpRun run : _loadedRuns) - { - String objectId = _runWorkflowTaskMap.get(run.getRowId()); - - if (objectId != null) - { - List protocolApplications = ExperimentService.get().getExpProtocolApplicationsByObjectId(getContainer(), objectId); - - if (protocolApplications.size() > 1) - { - throw new ExperimentException("Multiple ProtocolApplications found with object id: " + objectId); - } - else if (protocolApplications.isEmpty()) - { - getLog().warn("Could not find ProtocolApplication with LSID containing object id: " + objectId); - } - else - { - run.setWorkflowTaskId(protocolApplications.get(0).getRowId()); - - try { - run.save(getUser()); - } - catch (BatchValidationException e) - { - throw new ExperimentException(e); - } - } - } - } - } - public List getProcessedRunsLSIDs() { return _processedRunsLSIDs; @@ -1449,9 +1404,7 @@ private void loadProtocolApplication(ProtocolApplicationBaseType xmlProtocolApp, if ((protocolLSID.equals(SAMPLE_ALIQUOT_PROTOCOL_LSID) || protocolLSID.equals(SAMPLE_DERIVATION_PROTOCOL_LSID)) && !material.isOperationPermitted(SampleTypeService.SampleOperations.EditLineage)) return SampleTypeService.get().getOperationNotPermittedMessage(Collections.singleton(material), SampleTypeService.SampleOperations.EditLineage); - if ((ExpProtocol.isSampleWorkflowProtocol(protocolLSID)) - && !material.isOperationPermitted(SampleTypeService.SampleOperations.AddToWorkflow)) - return SampleTypeService.get().getOperationNotPermittedMessage(Collections.singleton(material), SampleTypeService.SampleOperations.AddToWorkflow); + return null; } diff --git a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java index 041d8a91675..9baee1183c3 100644 --- a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java @@ -855,6 +855,23 @@ public List getExpMaterialsByName(@NotNull String name, @Nullab .toList(); } + @NotNull + @Override + public List getExpMaterialsByName(@NotNull Collection names, @NotNull String sampleTypeName, @NotNull Container container, User user) + { + ExpSampleType sampleType = SampleTypeService.get().getSampleType(container, user, sampleTypeName); + if (sampleType == null) + return Collections.emptyList(); + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(ExpMaterialTable.Column.Name.name()), names, IN); + filter.addCondition(FieldKey.fromParts("Container"), container); + filter.addCondition(FieldKey.fromParts("CpasType"), sampleType.getLSID()); + + return getExpMaterials(filter) + .stream() + .filter(m -> m.getContainer().hasPermission(user, ReadPermission.class) && m.getSampleType() != null) + .toList(); + } + public @Nullable ExpMaterialImpl getExpMaterial(SimpleFilter filter) { Material material = new TableSelector(getTinfoMaterial(), filter, null).getObject(Material.class); diff --git a/experiment/src/org/labkey/experiment/samples/AbstractExpFolderImporter.java b/experiment/src/org/labkey/experiment/samples/AbstractExpFolderImporter.java index 5147ae170f4..0b446196ae0 100644 --- a/experiment/src/org/labkey/experiment/samples/AbstractExpFolderImporter.java +++ b/experiment/src/org/labkey/experiment/samples/AbstractExpFolderImporter.java @@ -157,7 +157,7 @@ else if (file.equalsIgnoreCase(XAR_RUNS_NAME) || file.equalsIgnoreCase(XAR_RUNS_ throw(e); } log.info("Importing the runs XAR file: " + runsXarFile.getName()); - XarReader runsReader = new FolderXarImporterFactory.FolderExportXarReader(runsXarSource, job); + XarReader runsReader = new FolderXarImporterFactory.FolderExportXarReader(runsXarSource, job, ctx); runsReader.setStrictValidateExistingSampleType(xarCtx.isStrictValidateExistingSampleType()); runsReader.parseAndLoad(false, ctx.getAuditBehaviorType()); } @@ -220,7 +220,7 @@ protected XarReader getXarReader(@Nullable PipelineJob job, FolderImportContext log.error("Failed to initialize types XAR source", e); throw(e); } - return new FolderXarImporterFactory.FolderExportXarReader(typesXarSource, job); + return new FolderXarImporterFactory.FolderExportXarReader(typesXarSource, job, ctx); } protected PipelineJob getDummyPipelineJob(FolderImportContext ctx) diff --git a/experiment/src/org/labkey/experiment/xar/FolderXarImporterFactory.java b/experiment/src/org/labkey/experiment/xar/FolderXarImporterFactory.java index 18326f029fd..003dc8fb0da 100644 --- a/experiment/src/org/labkey/experiment/xar/FolderXarImporterFactory.java +++ b/experiment/src/org/labkey/experiment/xar/FolderXarImporterFactory.java @@ -141,7 +141,7 @@ protected XarSource createXarSource(FileLike file) throw(e); } - FolderExportXarReader reader = new FolderExportXarReader(xarSource, job); + FolderExportXarReader reader = new FolderExportXarReader(xarSource, job, ctx); XarImportContext xarCtx = ctx.getContext(XarImportContext.class); if (xarCtx != null) { @@ -238,6 +238,11 @@ public FolderExportXarReader(XarSource source, PipelineJob job) super(source, job); } + public FolderExportXarReader(XarSource source, PipelineJob job, FolderImportContext ctx) + { + super(source, job, ctx); + } + @Override protected Container getContainer() { diff --git a/experiment/src/org/labkey/experiment/xar/FolderXarWriterFactory.java b/experiment/src/org/labkey/experiment/xar/FolderXarWriterFactory.java index dc71d2e758c..334f13fb22c 100644 --- a/experiment/src/org/labkey/experiment/xar/FolderXarWriterFactory.java +++ b/experiment/src/org/labkey/experiment/xar/FolderXarWriterFactory.java @@ -135,22 +135,8 @@ private boolean isETL(ExpRun run) private List getRuns(Container c) { - // Don't include the sample derivation runs; we now have a separate exporter explicitly for sample types. - // Also don't include recipe protocols; there's a separate folder writer and importer for the recipe module. - // if an additional context has been furnished, filter out runs not included in this export - List allRuns = ExperimentServiceImpl.get().getExpRuns(c, null, null, new RunFilter(c)); - // the smJobRuns can make reference to assay designs, so we will put all the SM Task and Protocols at the end to assure - // the assay definitions have already been processed and can be resolved properly. - List reorderedRuns = allRuns.stream() - .filter((run -> !ExpProtocol.isSampleWorkflowProtocol(run.getProtocol().getLSID()))) - .collect(Collectors.toList()); - List smJobRuns = allRuns.stream() - .filter((run -> ExpProtocol.isSampleWorkflowProtocol(run.getProtocol().getLSID()))) - .toList(); - - reorderedRuns.addAll(smJobRuns); - return reorderedRuns; + return new ArrayList<>(allRuns); } private List getProtocols(Container c) @@ -166,16 +152,8 @@ private List getProtocols(Container c) && !"recipe".equalsIgnoreCase(protocol.getLSIDNamespacePrefix()) ) .collect(Collectors.toList()); - // the sm template tasks can make reference to assay designs, so we will put all the SM Job and Task Protocols at the end to assure - // the assay definitions have already been processed and can be resolved properly. - List reorderedProtocols = protocols.stream() - .filter((protocol -> !ExpProtocol.isSampleWorkflowProtocol(protocol.getLSID())) - ) - .collect(Collectors.toList()); - protocols.stream() - .filter(protocol -> ExpProtocol.isSampleWorkflowProtocol(protocol.getLSID())) - .forEach(reorderedProtocols::add); - return reorderedProtocols.stream() + + return protocols.stream() .map(ExpObject::getRowId) .collect(Collectors.toList()); }