From ded45032ac82bd76cf70964087a56ee72f346dc7 Mon Sep 17 00:00:00 2001 From: cnathe Date: Fri, 3 Oct 2025 14:51:12 -0500 Subject: [PATCH 1/3] StoredAmount / RawAmount display formatting to be done with DisplayColumnFactory - use sample type precision for formatting if the columnInfo doesn't have a user defined format - remove ROUND() from StoredAmount ExprColumn since we need to have access to value and displayValue on client side --- .../experiment/api/ExpMaterialTableImpl.java | 3689 +++++++++-------- 1 file changed, 1856 insertions(+), 1833 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index 864eae8bf50..94d8fd0bfb0 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -1,1833 +1,1856 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.experiment.api; - -import org.apache.commons.collections4.ListUtils; -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.assay.plate.AssayPlateMetadataService; -import org.labkey.api.audit.AuditHandler; -import org.labkey.api.cache.BlockingCache; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.compliance.TableRules; -import org.labkey.api.compliance.TableRulesManager; -import org.labkey.api.data.ColumnHeaderType; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DataColumn; -import org.labkey.api.data.DataRegion; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.DisplayColumn; -import org.labkey.api.data.DisplayColumnFactory; -import org.labkey.api.data.ForeignKey; -import org.labkey.api.data.ImportAliasable; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.MaterializedQueryHelper; -import org.labkey.api.data.MutableColumnInfo; -import org.labkey.api.data.PHI; -import org.labkey.api.data.RenderContext; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.Sort; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.UnionContainerFilter; -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.dataiterator.DataIteratorContext; -import org.labkey.api.dataiterator.LoggingDataIterator; -import org.labkey.api.dataiterator.SimpleTranslator; -import org.labkey.api.exp.MvColumn; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.PropertyColumn; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.ExperimentUrls; -import org.labkey.api.exp.api.NameExpressionOptionService; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.DomainUtil; -import org.labkey.api.exp.query.ExpDataTable; -import org.labkey.api.exp.query.ExpMaterialTable; -import org.labkey.api.exp.query.ExpSampleTypeTable; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.exp.query.SamplesSchema; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.inventory.InventoryService; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.ontology.Unit; -import org.labkey.api.qc.SampleStatusService; -import org.labkey.api.query.AliasedColumn; -import org.labkey.api.query.DetailsURL; -import org.labkey.api.query.ExprColumn; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.LookupForeignKey; -import org.labkey.api.query.QueryException; -import org.labkey.api.query.QueryForeignKey; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QueryUpdateService; -import org.labkey.api.query.QueryUrls; -import org.labkey.api.query.RowIdForeignKey; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.UserSchema; -import org.labkey.api.query.column.BuiltInColumnTypes; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.UserPrincipal; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.InsertPermission; -import org.labkey.api.security.permissions.MediaReadPermission; -import org.labkey.api.security.permissions.MoveEntitiesPermission; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.util.GUID; -import org.labkey.api.util.HeartBeat; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.StringExpression; -import org.labkey.api.util.UnexpectedException; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.ViewContext; -import org.labkey.data.xml.TableType; -import org.labkey.experiment.ExpDataIterators; -import org.labkey.experiment.ExpDataIterators.AliasDataIteratorBuilder; -import org.labkey.experiment.controllers.exp.ExperimentController; -import org.labkey.experiment.lineage.LineageMethod; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static java.util.Objects.requireNonNull; -import static org.labkey.api.exp.api.SampleTypeDomainKind.ALIQUOT_COUNT_LABEL; -import static org.labkey.api.exp.api.SampleTypeDomainKind.ALIQUOT_VOLUME_LABEL; -import static org.labkey.api.exp.api.SampleTypeDomainKind.AVAILABLE_ALIQUOT_COUNT_LABEL; -import static org.labkey.api.exp.api.SampleTypeDomainKind.AVAILABLE_ALIQUOT_VOLUME_LABEL; -import static org.labkey.api.exp.api.SampleTypeDomainKind.SAMPLETYPE_FILE_DIRECTORY; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotCount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotVolume; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AvailableAliquotCount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AvailableAliquotVolume; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; -import static org.labkey.api.util.StringExpressionFactory.AbstractStringExpression.NullValueBehavior.NullResult; -import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.schema; - -public class ExpMaterialTableImpl extends ExpRunItemTableImpl implements ExpMaterialTable -{ - ExpSampleTypeImpl _ss; - Set _uniqueIdFields; - boolean _supportTableRules = true; - - public static final Set MATERIAL_ALT_MERGE_KEYS; - public static final Set MATERIAL_ALT_UPDATE_KEYS; - static { - MATERIAL_ALT_MERGE_KEYS = Set.of(Column.MaterialSourceId.name(), Column.Name.name()); - MATERIAL_ALT_UPDATE_KEYS = Set.of(Column.LSID.name()); - } - - public ExpMaterialTableImpl(UserSchema schema, ContainerFilter cf, @Nullable ExpSampleType sampleType) - { - super(ExpSchema.TableType.Materials.name(), ExperimentServiceImpl.get().getTinfoMaterial(), schema, cf); - setDetailsURL(new DetailsURL(new ActionURL(ExperimentController.ShowMaterialAction.class, schema.getContainer()), Collections.singletonMap("rowId", "rowId"), NullResult)); - setPublicSchemaName(ExpSchema.SCHEMA_NAME); - addAllowablePermission(InsertPermission.class); - addAllowablePermission(UpdatePermission.class); - addAllowablePermission(MoveEntitiesPermission.class); - setAllowedInsertOption(QueryUpdateService.InsertOption.MERGE); - setSampleType(sampleType); - } - - public Set getUniqueIdFields() - { - if (_uniqueIdFields == null) - { - _uniqueIdFields = new CaseInsensitiveHashSet(); - _uniqueIdFields.addAll(getColumns().stream().filter(ColumnInfo::isUniqueIdField).map(ColumnInfo::getName).collect(Collectors.toSet())); - } - return _uniqueIdFields; - } - - @Override - protected ColumnInfo resolveColumn(String name) - { - ColumnInfo result = super.resolveColumn(name); - if (result == null) - { - if ("CpasType".equalsIgnoreCase(name)) - result = createColumn(Column.SampleSet.name(), Column.SampleSet); - else if (Column.Property.name().equalsIgnoreCase(name)) - result = createPropertyColumn(Column.Property.name()); - else if (Column.QueryableInputs.name().equalsIgnoreCase(name)) - result = createColumn(Column.QueryableInputs.name(), Column.QueryableInputs); - } - return result; - } - - @Override - public ColumnInfo getExpObjectColumn() - { - var ret = wrapColumn("ExpMaterialTableImpl_object_", _rootTable.getColumn("objectid")); - ret.setConceptURI(BuiltInColumnTypes.EXPOBJECTID_CONCEPT_URI); - return ret; - } - - @Override - public AuditHandler getAuditHandler(AuditBehaviorType auditBehaviorType) - { - if (getUserSchema().getName().equalsIgnoreCase(SamplesSchema.SCHEMA_NAME)) - { - // Special case sample auditing to help build a useful timeline view - return SampleTypeServiceImpl.get(); - } - - return super.getAuditHandler(auditBehaviorType); - } - - @Override - public MutableColumnInfo createColumn(String alias, Column column) - { - switch (column) - { - case Folder -> - { - return wrapColumn(alias, _rootTable.getColumn("Container")); - } - case LSID -> - { - return wrapColumn(alias, _rootTable.getColumn(Column.LSID.name())); - } - case MaterialSourceId -> - { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.MaterialSourceId.name())); - columnInfo.setFk(new LookupForeignKey(getLookupContainerFilter(), null, null, null, null, "RowId", "Name") - { - @Override - public TableInfo getLookupTableInfo() - { - ExpSampleTypeTable sampleTypeTable = ExperimentService.get().createSampleTypeTable(ExpSchema.TableType.SampleSets.toString(), _userSchema, getLookupContainerFilter()); - sampleTypeTable.populate(); - return sampleTypeTable; - } - - @Override - public StringExpression getURL(ColumnInfo parent) - { - return super.getURL(parent, true); - } - }); - columnInfo.setUserEditable(false); - columnInfo.setReadOnly(true); - columnInfo.setHidden(true); - return columnInfo; - } - case RootMaterialRowId -> - { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.RootMaterialRowId.name())); - columnInfo.setFk(getExpSchema().getMaterialForeignKey(getLookupContainerFilter(), Column.RowId.name())); - columnInfo.setLabel("Root Material"); - columnInfo.setUserEditable(false); - - // NK: Here we mark the column as not required AND nullable which is the opposite of the database where - // a NOT NULL constraint is in place. This is done to avoid the RequiredValidator check upon updating a row. - // See ExpMaterialValidatorIterator. - columnInfo.setRequired(false); - columnInfo.setNullable(true); - - return columnInfo; - } - case AliquotedFromLSID -> - { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.AliquotedFromLSID.name())); - columnInfo.setSqlTypeName("lsidtype"); - columnInfo.setFk(getExpSchema().getMaterialForeignKey(getLookupContainerFilter(), Column.LSID.name())); - columnInfo.setLabel("Aliquoted From Parent"); - return columnInfo; - } - case IsAliquot -> - { - String rootMaterialRowIdField = ExprColumn.STR_TABLE_ALIAS + "." + Column.RootMaterialRowId.name(); - String rowIdField = ExprColumn.STR_TABLE_ALIAS + "." + Column.RowId.name(); - ExprColumn columnInfo = new ExprColumn(this, FieldKey.fromParts(Column.IsAliquot.name()), new SQLFragment( - "(CASE WHEN (" + rootMaterialRowIdField + " = " + rowIdField + ") THEN ").append(getSqlDialect().getBooleanFALSE()) - .append(" WHEN ").append(rowIdField).append(" IS NOT NULL THEN ").append(getSqlDialect().getBooleanTRUE()) // Issue 52745 - .append(" ELSE NULL END)"), JdbcType.BOOLEAN); - columnInfo.setLabel("Is Aliquot"); - columnInfo.setDescription("Identifies if the material is a sample or an aliquot"); - columnInfo.setUserEditable(false); - columnInfo.setReadOnly(true); - columnInfo.setHidden(false); - return columnInfo; - } - case Name -> - { - var nameCol = wrapColumn(alias, _rootTable.getColumn(column.toString())); - // shut off this field in insert and update views if user specified names are not allowed - if (!NameExpressionOptionService.get().getAllowUserSpecificNamesValue(getContainer())) - { - nameCol.setShownInInsertView(false); - nameCol.setShownInUpdateView(false); - } - return nameCol; - } - case RawAmount -> - { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.StoredAmount.name())); - columnInfo.setDescription("The amount of this sample, in the base unit for the sample type's display unit (if defined), currently on hand."); - if (columnInfo.getFormat() == null) - columnInfo.setFormat(Quantity.DEFAULT_FORMAT); - columnInfo.setUserEditable(false); - columnInfo.setReadOnly(true); - return columnInfo; - } - case StoredAmount -> - { - String label = StoredAmount.label(); - Set importAliases = Set.of(label, "Stored Amount"); - Unit typeUnit = getSampleTypeUnit(); - if (typeUnit != null) - { - SampleTypeAmountDisplayColumn columnInfo = new SampleTypeAmountDisplayColumn(this, Column.StoredAmount.name(), Column.Units.name(), label, importAliases, typeUnit); - columnInfo.setDescription("The amount of this sample, in the display unit for the sample type, currently on hand."); - columnInfo.setShownInUpdateView(true); - columnInfo.setShownInInsertView(true); - columnInfo.setUserEditable(true); - columnInfo.setCalculated(false); - return columnInfo; - } - else - { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.StoredAmount.name())); - if (columnInfo.getFormat() == null) - columnInfo.setFormat(Quantity.DEFAULT_FORMAT); - columnInfo.setLabel(label); - columnInfo.setImportAliasesSet(importAliases); - columnInfo.setDescription("The amount of this sample currently on hand."); - return columnInfo; - } - } - case RawUnits -> - { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.Units.name())); - columnInfo.setDescription("The units associated with the Stored Amount for this sample."); - columnInfo.setUserEditable(false); - columnInfo.setReadOnly(true); - return columnInfo; - } - case Units -> - { - ForeignKey fk = new LookupForeignKey("Value", "Value") - { - @Override - public @Nullable TableInfo getLookupTableInfo() - { - return getExpSchema().getTable(ExpSchema.MEASUREMENT_UNITS_TABLE); - } - }; - - Unit typeUnit = getSampleTypeUnit(); - if (typeUnit != null) - { - SampleTypeUnitDisplayColumn columnInfo = new SampleTypeUnitDisplayColumn(this, Column.Units.name(), typeUnit); - columnInfo.setFk(fk); - columnInfo.setDescription("The sample type display units associated with the Amount for this sample."); - columnInfo.setShownInUpdateView(true); - columnInfo.setShownInInsertView(true); - columnInfo.setUserEditable(true); - columnInfo.setCalculated(false); - return columnInfo; - } - else - { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.Units.name())); - columnInfo.setFk(fk); - columnInfo.setDescription("The units associated with the Stored Amount for this sample."); - return columnInfo; - } - } - case Description -> - { - return wrapColumn(alias, _rootTable.getColumn(Column.Description.name())); - } - case SampleSet -> - { - var columnInfo = wrapColumn(alias, _rootTable.getColumn("CpasType")); - // NOTE: populateColumns() overwrites this with a QueryForeignKey. Can this be removed? - columnInfo.setFk(new LookupForeignKey(getContainerFilter(), null, null, null, null, "LSID", "Name") - { - @Override - public TableInfo getLookupTableInfo() - { - ExpSampleTypeTable sampleTypeTable = ExperimentService.get().createSampleTypeTable(ExpSchema.TableType.SampleSets.toString(), _userSchema, getLookupContainerFilter()); - sampleTypeTable.populate(); - return sampleTypeTable; - } - - @Override - public StringExpression getURL(ColumnInfo parent) - { - return super.getURL(parent, true); - } - }); - return columnInfo; - } - case SourceProtocolLSID -> - { - // NOTE: This column is incorrectly named "Protocol", but we are keeping it for backwards compatibility to avoid breaking queries in hvtnFlow module - ExprColumn columnInfo = new ExprColumn(this, ExpDataTable.Column.Protocol.toString(), new SQLFragment( - "(SELECT ProtocolLSID FROM " + ExperimentServiceImpl.get().getTinfoProtocolApplication() + " pa " + - " WHERE pa.RowId = " + ExprColumn.STR_TABLE_ALIAS + ".SourceApplicationId)"), JdbcType.VARCHAR); - columnInfo.setSqlTypeName("lsidtype"); - columnInfo.setFk(getExpSchema().getProtocolForeignKey(getContainerFilter(), "LSID")); - columnInfo.setLabel("Source Protocol"); - columnInfo.setDescription("Contains a reference to the protocol for the protocol application that created this sample"); - columnInfo.setUserEditable(false); - columnInfo.setReadOnly(true); - columnInfo.setHidden(true); - return columnInfo; - } - case SourceProtocolApplication -> - { - var columnInfo = wrapColumn(alias, _rootTable.getColumn("SourceApplicationId")); - columnInfo.setFk(getExpSchema().getProtocolApplicationForeignKey(getContainerFilter())); - columnInfo.setUserEditable(false); - columnInfo.setReadOnly(true); - columnInfo.setHidden(true); - columnInfo.setAutoIncrement(false); - return columnInfo; - } - case SourceApplicationInput -> - { - var col = createEdgeColumn(alias, Column.SourceProtocolApplication, ExpSchema.TableType.MaterialInputs); - col.setDescription("Contains a reference to the MaterialInput row between this ExpMaterial and it's SourceProtocolApplication"); - col.setHidden(true); - return col; - } - case RunApplication -> - { - SQLFragment sql = new SQLFragment("(SELECT pa.rowId FROM ") - .append(ExperimentService.get().getTinfoProtocolApplication(), "pa") - .append(" WHERE pa.runId = ").append(ExprColumn.STR_TABLE_ALIAS).append(".runId") - .append(" AND pa.cpasType = ").appendValue(ExpProtocol.ApplicationType.ExperimentRunOutput) - .append(")"); - - var col = new ExprColumn(this, alias, sql, JdbcType.INTEGER); - col.setFk(getExpSchema().getProtocolApplicationForeignKey(getContainerFilter())); - col.setDescription("Contains a reference to the ExperimentRunOutput protocol application of the run that created this sample"); - col.setUserEditable(false); - col.setReadOnly(true); - col.setHidden(true); - return col; - } - case RunApplicationOutput -> - { - var col = createEdgeColumn(alias, Column.RunApplication, ExpSchema.TableType.MaterialInputs); - col.setDescription("Contains a reference to the MaterialInput row between this ExpMaterial and it's RunOutputApplication"); - return col; - } - case Run -> - { - var ret = wrapColumn(alias, _rootTable.getColumn("RunId")); - ret.setReadOnly(true); - return ret; - } - case RowId -> - { - var ret = wrapColumn(alias, _rootTable.getColumn("RowId")); - // When no sorts are added by views, QueryServiceImpl.createDefaultSort() adds the primary key's default sort direction - ret.setSortDirection(Sort.SortDirection.DESC); - ret.setFk(new RowIdForeignKey(ret)); - ret.setUserEditable(false); - ret.setHidden(true); - ret.setShownInInsertView(false); - ret.setHasDbSequence(true); - ret.setIsRootDbSequence(true); - return ret; - } - case Property -> - { - return createPropertyColumn(alias); - } - case Flag -> - { - return createFlagColumn(alias); - } - case Created -> - { - return wrapColumn(alias, _rootTable.getColumn("Created")); - } - case CreatedBy -> - { - return createUserColumn(alias, _rootTable.getColumn("CreatedBy")); - } - case Modified -> - { - return wrapColumn(alias, _rootTable.getColumn("Modified")); - } - case ModifiedBy -> - { - return createUserColumn(alias, _rootTable.getColumn("ModifiedBy")); - } - case Alias -> - { - return createAliasColumn(alias, ExperimentService.get()::getTinfoMaterialAliasMap); - } - case Inputs -> - { - return createLineageColumn(this, alias, true, false); - } - case QueryableInputs -> - { - return createLineageColumn(this, alias, true, true); - } - case Outputs -> - { - return createLineageColumn(this, alias, false, false); - } - case Properties -> - { - return createPropertiesColumn(alias); - } - case SampleState -> - { - boolean statusEnabled = SampleStatusService.get().supportsSampleStatus() && !SampleStatusService.get().getAllProjectStates(getContainer()).isEmpty(); - var ret = wrapColumn(alias, _rootTable.getColumn(column.name())); - ret.setLabel("Status"); - ret.setHidden(!statusEnabled); - ret.setShownInDetailsView(statusEnabled); - ret.setShownInInsertView(statusEnabled); - ret.setShownInUpdateView(statusEnabled); - ret.setRemapMissingBehavior(SimpleTranslator.RemapMissingBehavior.Error); - ret.setFk(new QueryForeignKey.Builder(getUserSchema(), getSampleStatusLookupContainerFilter()) - .schema(getExpSchema()).table(ExpSchema.TableType.SampleStatus).display("Label")); - return ret; - } - case AliquotCount -> - { - var ret = wrapColumn(alias, _rootTable.getColumn(AliquotCount.name())); - ret.setLabel(ALIQUOT_COUNT_LABEL); - return ret; - } - case AliquotVolume -> - { - var ret = wrapColumn(alias, _rootTable.getColumn(AliquotVolume.name())); - ret.setLabel(ALIQUOT_VOLUME_LABEL); - return ret; - } - case AvailableAliquotVolume -> - { - var ret = wrapColumn(alias, _rootTable.getColumn(AvailableAliquotVolume.name())); - ret.setLabel(AVAILABLE_ALIQUOT_VOLUME_LABEL); - return ret; - } - case AvailableAliquotCount -> - { - var ret = wrapColumn(alias, _rootTable.getColumn(AvailableAliquotCount.name())); - ret.setLabel(AVAILABLE_ALIQUOT_COUNT_LABEL); - return ret; - } - case AliquotUnit -> - { - var ret = wrapColumn(alias, _rootTable.getColumn("AliquotUnit")); - ret.setShownInDetailsView(false); - return ret; - } - case MaterialExpDate -> - { - var ret = wrapColumn(alias, _rootTable.getColumn("MaterialExpDate")); - ret.setLabel("Expiration Date"); - ret.setShownInDetailsView(true); - ret.setShownInInsertView(true); - ret.setShownInUpdateView(true); - return ret; - } - default -> throw new IllegalArgumentException("Unknown column " + column); - } - } - - @Override - public MutableColumnInfo createPropertyColumn(String alias) - { - var ret = super.createPropertyColumn(alias); - if (_ss != null) - { - final TableInfo t = _ss.getTinfo(); - if (t != null) - { - ret.setFk(new LookupForeignKey() - { - @Override - public TableInfo getLookupTableInfo() - { - return t; - } - - @Override - protected ColumnInfo getPkColumn(TableInfo table) - { - return t.getColumn("lsid"); - } - }); - } - } - ret.setIsUnselectable(true); - ret.setDescription("A holder for any custom fields associated with this sample"); - ret.setHidden(true); - return ret; - } - - private Unit getSampleTypeUnit() - { - Unit typeUnit = null; - if (_ss != null && _ss.getMetricUnit() != null) - typeUnit = Unit.fromName(_ss.getMetricUnit()); - return typeUnit; - } - - private void setSampleType(@Nullable ExpSampleType st) - { - checkLocked(); - if (_ss != null) - { - throw new IllegalStateException("Cannot unset sample type"); - } - if (st != null && !(st instanceof ExpSampleTypeImpl)) - { - throw new IllegalArgumentException("Expected sample type to be an instance of " + ExpSampleTypeImpl.class.getName() + " but was a " + st.getClass().getName()); - } - _ss = (ExpSampleTypeImpl) st; - if (_ss != null) - { - setPublicSchemaName(SamplesSchema.SCHEMA_NAME); - setName(st.getName()); - - String description = _ss.getDescription(); - if (StringUtils.isEmpty(description)) - description = "Contains one row per sample in the " + _ss.getName() + " sample type"; - setDescription(description); - - if (canUserAccessPhi()) - { - ActionURL url = PageFlowUtil.urlProvider(ExperimentUrls.class).getImportSamplesURL(getContainer(), _ss.getName()); - setImportURL(new DetailsURL(url)); - } - } - } - - public ExpSampleType getSampleType() - { - return _ss; - } - - @Override - protected void populateColumns() - { - var st = getSampleType(); - var rowIdCol = addColumn(Column.RowId); - addColumn(Column.MaterialSourceId); - addColumn(Column.SourceProtocolApplication); - addColumn(Column.SourceApplicationInput); - addColumn(Column.RunApplication); - addColumn(Column.RunApplicationOutput); - addColumn(Column.SourceProtocolLSID); - - var nameCol = addColumn(Column.Name); - if (st != null && st.hasNameAsIdCol()) - { - // Show the Name field but don't mark is as required when using name expressions - if (st.hasNameExpression()) - { - var nameExpression = st.getNameExpression(); - nameCol.setNameExpression(nameExpression); - nameCol.setNullable(true); - String nameExpressionPreview = getExpNameExpressionPreview(getUserSchema().getSchemaName(), st.getName(), getUserSchema().getUser()); - String desc = appendNameExpressionDescription(nameCol.getDescription(), nameExpression, nameExpressionPreview); - nameCol.setDescription(desc); - } - else - { - nameCol.setNullable(false); - } - } - else - { - nameCol.setReadOnly(true); - nameCol.setShownInInsertView(false); - } - - addColumn(Column.Alias); - addColumn(Column.Description); - - var typeColumnInfo = addColumn(Column.SampleSet); - typeColumnInfo.setFk(new QueryForeignKey(_userSchema, getContainerFilter(), ExpSchema.SCHEMA_NAME, getContainer(), null, ExpSchema.TableType.SampleSets.name(), "lsid", null) - { - @Override - protected ContainerFilter getLookupContainerFilter() - { - // Be sure that we can resolve the sample type if it's defined in a separate container. - // Same as CurrentPlusProjectAndShared but includes SampleSet's container as well. - // Issue 37982: Sample Type: Link to precursor sample type does not resolve correctly if sample has - // parents in current sample type and a sample type in the parent container - Set containers = new HashSet<>(); - if (null != st) - containers.add(st.getContainer()); - containers.add(getContainer()); - if (getContainer().getProject() != null) - containers.add(getContainer().getProject()); - containers.add(ContainerManager.getSharedContainer()); - ContainerFilter cf = new ContainerFilter.CurrentPlusExtras(_userSchema.getContainer(), _userSchema.getUser(), containers); - - if (null != _containerFilter && _containerFilter.getType() != ContainerFilter.Type.Current) - cf = new UnionContainerFilter(_containerFilter, cf); - return cf; - } - }); - - typeColumnInfo.setReadOnly(true); - typeColumnInfo.setUserEditable(false); - typeColumnInfo.setShownInInsertView(false); - - addColumn(Column.MaterialExpDate); - addContainerColumn(Column.Folder, null); - var runCol = addColumn(Column.Run); - runCol.setFk(new ExpSchema(_userSchema.getUser(), getContainer()).getRunIdForeignKey(getContainerFilter())); - runCol.setShownInInsertView(false); - runCol.setShownInUpdateView(false); - - var colLSID = addColumn(Column.LSID); - colLSID.setHidden(true); - colLSID.setReadOnly(true); - colLSID.setUserEditable(false); - colLSID.setShownInInsertView(false); - colLSID.setShownInDetailsView(false); - colLSID.setShownInUpdateView(false); - - var rootRowId = addColumn(Column.RootMaterialRowId); - rootRowId.setHidden(true); - rootRowId.setReadOnly(true); - rootRowId.setUserEditable(false); - rootRowId.setShownInInsertView(false); - rootRowId.setShownInDetailsView(false); - rootRowId.setShownInUpdateView(false); - - var aliquotParentLSID = addColumn(Column.AliquotedFromLSID); - aliquotParentLSID.setHidden(true); - aliquotParentLSID.setReadOnly(true); - aliquotParentLSID.setUserEditable(false); - aliquotParentLSID.setShownInInsertView(false); - aliquotParentLSID.setShownInDetailsView(false); - aliquotParentLSID.setShownInUpdateView(false); - - addColumn(Column.IsAliquot); - addColumn(Column.Created); - addColumn(Column.CreatedBy); - addColumn(Column.Modified); - addColumn(Column.ModifiedBy); - - List defaultCols = new ArrayList<>(); - defaultCols.add(FieldKey.fromParts(Column.Name)); - defaultCols.add(FieldKey.fromParts(Column.MaterialExpDate)); - boolean hasProductFolders = getContainer().hasProductFolders(); - if (hasProductFolders) - defaultCols.add(FieldKey.fromParts(Column.Folder)); - defaultCols.add(FieldKey.fromParts(Column.Run)); - - if (st == null) - defaultCols.add(FieldKey.fromParts(Column.SampleSet)); - - addColumn(Column.Flag); - - var statusColInfo = addColumn(Column.SampleState); - boolean statusEnabled = SampleStatusService.get().supportsSampleStatus() && !SampleStatusService.get().getAllProjectStates(getContainer()).isEmpty(); - statusColInfo.setShownInDetailsView(statusEnabled); - statusColInfo.setShownInInsertView(statusEnabled); - statusColInfo.setShownInUpdateView(statusEnabled); - statusColInfo.setHidden(!statusEnabled); - statusColInfo.setRemapMissingBehavior(SimpleTranslator.RemapMissingBehavior.Error); - if (statusEnabled) - defaultCols.add(FieldKey.fromParts(Column.SampleState)); - statusColInfo.setFk(new QueryForeignKey.Builder(getUserSchema(), getSampleStatusLookupContainerFilter()) - .schema(getExpSchema()).table(ExpSchema.TableType.SampleStatus).display("Label")); - - // TODO is this a real Domain??? - if (st != null && !"urn:lsid:labkey.com:SampleSource:Default".equals(st.getDomain().getTypeURI())) - { - defaultCols.add(FieldKey.fromParts(Column.Flag)); - addSampleTypeColumns(st, defaultCols); - - setName(_ss.getName()); - - ActionURL gridUrl = new ActionURL(ExperimentController.ShowSampleTypeAction.class, getContainer()); - gridUrl.addParameter("rowId", st.getRowId()); - setGridURL(new DetailsURL(gridUrl)); - } - - List calculatedFieldKeys = DomainUtil.getCalculatedFieldsForDefaultView(this); - defaultCols.addAll(calculatedFieldKeys); - - addColumn(Column.AliquotCount); - addColumn(Column.AliquotVolume); - addColumn(Column.AliquotUnit); - addColumn(Column.AvailableAliquotCount); - addColumn(Column.AvailableAliquotVolume); - - addColumn(Column.StoredAmount); - defaultCols.add(FieldKey.fromParts(Column.StoredAmount)); - - addColumn(Column.Units); - defaultCols.add(FieldKey.fromParts(Column.Units)); - - var rawAmountColumn = addColumn(Column.RawAmount); - rawAmountColumn.setDisplayColumnFactory(new DisplayColumnFactory() - { - @Override - public DisplayColumn createRenderer(ColumnInfo colInfo) - { - return new DataColumn(colInfo) - { - @Override - public void addQueryFieldKeys(Set keys) - { - super.addQueryFieldKeys(keys); - keys.add(FieldKey.fromParts(Column.StoredAmount)); - - } - }; - } - }); - rawAmountColumn.setHidden(true); - rawAmountColumn.setShownInDetailsView(false); - rawAmountColumn.setShownInInsertView(false); - rawAmountColumn.setShownInUpdateView(false); - - var rawUnitsColumn = addColumn(Column.RawUnits); - rawUnitsColumn.setDisplayColumnFactory(new DisplayColumnFactory() - { - @Override - public DisplayColumn createRenderer(ColumnInfo colInfo) - { - return new DataColumn(colInfo) - { - @Override - public void addQueryFieldKeys(Set keys) - { - super.addQueryFieldKeys(keys); - keys.add(FieldKey.fromParts(Column.Units)); - - } - }; - } - }); - rawUnitsColumn.setHidden(true); - rawUnitsColumn.setShownInDetailsView(false); - rawUnitsColumn.setShownInInsertView(false); - rawUnitsColumn.setShownInUpdateView(false); - - if (InventoryService.get() != null && (st == null || !st.isMedia())) - defaultCols.addAll(InventoryService.get().addInventoryStatusColumns(st == null ? null : st.getMetricUnit(), this, getContainer(), _userSchema.getUser())); - - SQLFragment sql; - UserSchema plateUserSchema; - // Issue 53194 : this would be the case for linked to study samples. The contextual role is set up from the study dataset - // for the source sample, we want to allow the plate schema to inherit any contextual roles to allow querying - // against tables in that schema. - if (_userSchema instanceof UserSchema.HasContextualRoles samplesSchema && !samplesSchema.getContextualRoles().isEmpty()) - plateUserSchema = AssayPlateMetadataService.get().getPlateSchema(_userSchema, samplesSchema.getContextualRoles()); - else - plateUserSchema = QueryService.get().getUserSchema(_userSchema.getUser(), _userSchema.getContainer(), "plate"); - - if (plateUserSchema != null && plateUserSchema.getTable("Well") != null) - { - String rowIdField = ExprColumn.STR_TABLE_ALIAS + "." + Column.RowId.name(); - SQLFragment existsSubquery = new SQLFragment() - .append("SELECT 1 FROM ") - .append(plateUserSchema.getTable("Well"), "well") - .append(" WHERE well.sampleid = ").append(rowIdField); - - sql = new SQLFragment() - .append("CASE WHEN EXISTS (") - .append(existsSubquery) - .append(") THEN 'Plated' ") - .append("WHEN ").append(ExprColumn.STR_TABLE_ALIAS).append(".RowId").append(" IS NOT NULL THEN 'Not Plated' ")// Issue 52745 - .append("ELSE NULL END"); - } - else - { - sql = new SQLFragment("(SELECT NULL)"); - } - var col = new ExprColumn(this, Column.IsPlated.name(), sql, JdbcType.VARCHAR); - col.setDescription("Whether the sample that has been plated, if plating is supported."); - col.setUserEditable(false); - col.setReadOnly(true); - col.setShownInDetailsView(false); - col.setShownInInsertView(false); - col.setShownInUpdateView(false); - if (plateUserSchema != null) - col.setURL(DetailsURL.fromString("plate-isPlated.api?sampleId=${" + Column.RowId.name() + "}")); - addColumn(col); - - addVocabularyDomains(); - - addColumn(Column.Properties); - - var colInputs = addColumn(Column.Inputs); - addMethod("Inputs", new LineageMethod(colInputs, true), Set.of(colInputs.getFieldKey())); - - var colOutputs = addColumn(Column.Outputs); - addMethod("Outputs", new LineageMethod(colOutputs, false), Set.of(colOutputs.getFieldKey())); - - addExpObjectMethod(); - - ActionURL detailsUrl = new ActionURL(ExperimentController.ShowMaterialAction.class, getContainer()); - DetailsURL url = new DetailsURL(detailsUrl, Collections.singletonMap("rowId", "RowId"), NullResult); - nameCol.setURL(url); - rowIdCol.setURL(url); - setDetailsURL(url); - - if (canUserAccessPhi()) - { - ActionURL updateActionURL = PageFlowUtil.urlProvider(ExperimentUrls.class).getUpdateMaterialQueryRowAction(getContainer(), this); - setUpdateURL(new DetailsURL(updateActionURL, Collections.singletonMap("RowId", "RowId"))); - - ActionURL insertActionURL = PageFlowUtil.urlProvider(ExperimentUrls.class).getInsertMaterialQueryRowAction(getContainer(), this); - setInsertURL(new DetailsURL(insertActionURL)); - } - else - { - setImportURL(LINK_DISABLER); - setInsertURL(LINK_DISABLER); - setUpdateURL(LINK_DISABLER); - } - - setTitleColumn(Column.Name.toString()); - - setDefaultVisibleColumns(defaultCols); - - MutableColumnInfo lineageLookup = ClosureQueryHelper.createAncestorLookupColumnInfo("Ancestors", this, _rootTable.getColumn("rowid"), _ss, true); - addColumn(lineageLookup); - } - - private ContainerFilter getSampleStatusLookupContainerFilter() - { - // The default lookup container filter is Current, but we want to have the default be CurrentPlusProjectAndShared - // for the sample status lookup since in the app project context we want to share status definitions across - // a given project instead of creating duplicate statuses in each subfolder project. - ContainerFilter.Type type = QueryService.get().getContainerFilterTypeForLookups(getContainer()); - type = type == null ? ContainerFilter.Type.CurrentPlusProjectAndShared : type; - return type.create(getUserSchema()); - } - - @Override - public Domain getDomain() - { - return getDomain(false); - } - - @Override - public Domain getDomain(boolean forUpdate) - { - return _ss == null ? null : _ss.getDomain(forUpdate); - } - - - public static String appendNameExpressionDescription(String currentDescription, String nameExpression, String nameExpressionPreview) - { - if (nameExpression == null) - return currentDescription; - - StringBuilder sb = new StringBuilder(); - if (currentDescription != null && !currentDescription.isEmpty()) - { - sb.append(currentDescription); - if (!currentDescription.endsWith(".")) - sb.append("."); - sb.append("\n"); - } - - sb.append("\nIf not provided, a unique name will be generated from the expression:\n"); - sb.append(nameExpression); - sb.append("."); - if (!StringUtils.isEmpty(nameExpressionPreview)) - { - sb.append("\nExample of name that will be generated from the current pattern: \n"); - sb.append(nameExpressionPreview); - } - - return sb.toString(); - } - - private void addSampleTypeColumns(ExpSampleType st, List visibleColumns) - { - TableInfo dbTable = ((ExpSampleTypeImpl)st).getTinfo(); - if (null == dbTable) - return; - - UserSchema schema = getUserSchema(); - Domain domain = st.getDomain(); - ColumnInfo rowIdColumn = getColumn(Column.RowId); - ColumnInfo lsidColumn = getColumn(Column.LSID); - ColumnInfo nameColumn = getColumn(Column.Name); - - visibleColumns.remove(FieldKey.fromParts(Column.Run.name())); - - // When not using name expressions, mark the ID columns as required. - // NOTE: If not explicitly set, the first domain property will be chosen as the ID column. - final List idCols = st.hasNameExpression() ? Collections.emptyList() : st.getIdCols(); - - Set mvColumns = domain.getProperties().stream() - .filter(ImportAliasable::isMvEnabled) - .map(dp -> FieldKey.fromParts(dp.getPropertyDescriptor().getMvIndicatorStorageColumnName())) - .collect(Collectors.toSet()); - - for (ColumnInfo dbColumn : dbTable.getColumns()) - { - // Don't include PHI columns in full text search index - // CONSIDER: Can we move this to a base class? Maybe in .addColumn() - if (schema.getUser().isSearchUser() && !dbColumn.getPHI().isLevelAllowed(PHI.NotPHI)) - continue; - - if ( - rowIdColumn.getFieldKey().equals(dbColumn.getFieldKey()) || - lsidColumn.getFieldKey().equals(dbColumn.getFieldKey()) || - nameColumn.getFieldKey().equals(dbColumn.getFieldKey()) - ) - { - continue; - } - - var wrapped = wrapColumnFromJoinedTable(dbColumn.getName(), dbColumn); - - // TODO missing values? comments? flags? - DomainProperty dp = domain.getPropertyByURI(dbColumn.getPropertyURI()); - var propColumn = copyColumnFromJoinedTable(null==dp ? dbColumn.getName() : dp.getName(), wrapped); - if (propColumn.getName().equalsIgnoreCase("genid")) - { - propColumn.setHidden(true); - propColumn.setUserEditable(false); - propColumn.setShownInDetailsView(false); - propColumn.setShownInInsertView(false); - propColumn.setShownInUpdateView(false); - } - if (null != dp) - { - PropertyColumn.copyAttributes(schema.getUser(), propColumn, dp.getPropertyDescriptor(), schema.getContainer(), - SchemaKey.fromParts("samples"), st.getName(), FieldKey.fromParts("RowId"), null, getLookupContainerFilter()); - - if (idCols.contains(dp)) - { - propColumn.setNullable(false); - propColumn.setDisplayColumnFactory(new IdColumnRendererFactory()); - } - - // Issue 38341: domain designer advanced settings 'show in default view' setting is not respected - if (!propColumn.isHidden()) - { - visibleColumns.add(propColumn.getFieldKey()); - } - - if (propColumn.isMvEnabled()) - { - // The column in the physical table has a "_MVIndicator" suffix, but we want to expose - // it with a "MVIndicator" suffix (no underscore) - var mvColumn = new AliasedColumn(this, dp.getName() + MvColumn.MV_INDICATOR_SUFFIX, - StorageProvisioner.get().getMvIndicatorColumn(dbTable, dp.getPropertyDescriptor(), "No MV column found for '" + dp.getName() + "' in sample type '" + getName() + "'")); - mvColumn.setLabel(dp.getLabel() != null ? dp.getLabel() : dp.getName() + " MV Indicator"); - mvColumn.setSqlTypeName("VARCHAR"); - mvColumn.setPropertyURI(dp.getPropertyURI()); - mvColumn.setNullable(true); - mvColumn.setUserEditable(false); - mvColumn.setHidden(true); - mvColumn.setMvIndicatorColumn(true); - - addColumn(mvColumn); - propColumn.setMvColumnName(FieldKey.fromParts(dp.getName() + MvColumn.MV_INDICATOR_SUFFIX)); - } - } - - if (!mvColumns.contains(propColumn.getFieldKey())) - addColumn(propColumn); - - } - - setDefaultVisibleColumns(visibleColumns); - } - - // These are mostly fields that are wrapped by fields with different names (see createColumn()) - // we could handle each case separately, but this is easier - static final Set wrappedFieldKeys = Set.of( - new FieldKey(null, "objectid"), - new FieldKey(null, "RowId"), - new FieldKey(null, "LSID"), // Flag - new FieldKey(null, "SourceApplicationId"), // SourceProtocolApplication - new FieldKey(null, "runId"), // Run, RunApplication - new FieldKey(null, "CpasType")); // SampleSet - static final Set ALL_COLUMNS = Set.of(); - - private @NotNull Set computeInnerSelectedColumns(Set selectedColumns) - { - if (null == selectedColumns) - return ALL_COLUMNS; - selectedColumns = new TreeSet<>(selectedColumns); - if (selectedColumns.contains(new FieldKey(null, StoredAmount))) - selectedColumns.add(new FieldKey(null, Units)); - if (selectedColumns.contains(new FieldKey(null, ExpMaterial.ALIQUOTED_FROM_INPUT))) - selectedColumns.add(new FieldKey(null, Column.AliquotedFromLSID.name())); - if (selectedColumns.contains(new FieldKey(null, Column.IsAliquot.name()))) - selectedColumns.add(new FieldKey(null, Column.RootMaterialRowId.name())); - selectedColumns.addAll(wrappedFieldKeys); - if (null != getFilter()) - selectedColumns.addAll(getFilter().getAllFieldKeys()); - return selectedColumns; - } - - @NotNull - @Override - public SQLFragment getFromSQL(String alias) - { - return getFromSQL(alias, null); - } - - @Override - public SQLFragment getFromSQLExpanded(String alias, Set selectedColumns) - { - SQLFragment sql = new SQLFragment("("); - boolean usedMaterialized; - - - // SELECT FROM - /* NOTE We want to avoid caching in paths where the table is actively being updated (e.g. loadRows) - * Unfortunately, we don't _really_ know when this is, but if we in a transaction that's a good guess. - * Also, we may use RemapCache for material lookup outside a transaction - */ - boolean onlyMaterialColums = false; - if (null != selectedColumns && !selectedColumns.isEmpty()) - onlyMaterialColums = selectedColumns.stream().allMatch(fk -> fk.getName().equalsIgnoreCase("Folder") || null != _rootTable.getColumn(fk)); - if (!onlyMaterialColums && null != _ss && null != _ss.getTinfo() && !getExpSchema().getDbSchema().getScope().isTransactionActive()) - { - sql.append(getMaterializedSQL()); - usedMaterialized = true; - } - else - { - sql.append(getJoinSQL(selectedColumns)); - usedMaterialized = false; - } - - // WHERE - SQLFragment filterFrag = getFilter().getSQLFragment(_rootTable, null); - sql.append("\n").append(filterFrag); - if (_ss != null && !usedMaterialized) - { - if (!filterFrag.isEmpty()) - sql.append(" AND "); - else - sql.append(" WHERE "); - sql.append("CpasType = ").appendValue(_ss.getLSID()); - } - sql.append(") ").appendIdentifier(alias); - - return getTransformedFromSQL(sql); - } - - @Override - public void setSupportTableRules(boolean b) - { - this._supportTableRules = b; - } - - @Override - public boolean supportTableRules() // intentional override - { - return _supportTableRules; - } - - @Override - protected @NotNull TableRules findTableRules() - { - Container definitionContainer = getUserSchema().getContainer(); - if (null != _ss) - definitionContainer = _ss.getContainer(); - return TableRulesManager.get().getTableRules(definitionContainer, getUserSchema().getUser(), getUserSchema().getContainer()); - } - - - static class InvalidationCounters - { - public final AtomicLong update, insert, delete, rollup; - InvalidationCounters() - { - long l = System.currentTimeMillis(); - update = new AtomicLong(l); - insert = new AtomicLong(l); - delete = new AtomicLong(l); - rollup = new AtomicLong(l); - } - } - - static final BlockingCache _materializedQueries = CacheManager.getBlockingStringKeyCache(CacheManager.UNLIMITED, CacheManager.HOUR, "materialized sample types", null); - static final Map _invalidationCounters = Collections.synchronizedMap(new HashMap<>()); - static final AtomicBoolean initializedListeners = new AtomicBoolean(false); - - // used by SampleTypeServiceImpl.refreshSampleTypeMaterializedView() - public static void refreshMaterializedView(final String lsid, SampleTypeServiceImpl.SampleChangeType reason) - { - var scope = ExperimentServiceImpl.getExpSchema().getScope(); - var runnable = new RefreshMaterializedViewRunnable(lsid, reason); - scope.addCommitTask(runnable, DbScope.CommitTaskOption.POSTCOMMIT); - } - - private static class RefreshMaterializedViewRunnable implements Runnable - { - private final String _lsid; - private final SampleTypeServiceImpl.SampleChangeType _reason; - - public RefreshMaterializedViewRunnable(String lsid, SampleTypeServiceImpl.SampleChangeType reason) - { - _lsid = lsid; - _reason = reason; - } - - @Override - public void run() - { - if (_reason == schema) - { - /* NOTE: MaterializedQueryHelper can detect data changes and refresh the materialized view using the provided SQL. - * It does not handle schema changes where the SQL itself needs to be updated. In this case, we remove the - * MQH from the cache to force the SQL to be regenerated. - */ - _materializedQueries.remove(_lsid); - return; - } - var counters = getInvalidateCounters(_lsid); - switch (_reason) - { - case insert -> counters.insert.incrementAndGet(); - case rollup -> counters.rollup.incrementAndGet(); - case update -> counters.update.incrementAndGet(); - case delete -> counters.delete.incrementAndGet(); - default -> throw new IllegalStateException("Unexpected value: " + _reason); - } - } - - @Override - public boolean equals(Object obj) - { - return obj instanceof RefreshMaterializedViewRunnable other && _lsid.equals(other._lsid) && _reason.equals(other._reason); - } - } - - private static InvalidationCounters getInvalidateCounters(String lsid) - { - if (!initializedListeners.getAndSet(true)) - { - CacheManager.addListener(_invalidationCounters::clear); - } - return _invalidationCounters.computeIfAbsent(lsid, (unused) -> - new InvalidationCounters() - ); - } - - /* SELECT and JOIN, does not include WHERE, same as getJoinSQL() */ - private SQLFragment getMaterializedSQL() - { - if (null == _ss) - return getJoinSQL(null); - - var mqh = _materializedQueries.get(_ss.getLSID(), null, (unusedKey, unusedArg) -> - { - /* NOTE: MaterializedQueryHelper does have a pattern to help with detecting schema changes. - * Previously it has been used on non-provisioned tables. It might be helpful to have a pattern, - * even if just to help with race-conditions. - * - * Maybe have a callback to generate the SQL dynamically, and verify that the sql is unchanged. - */ - SQLFragment viewSql = getJoinSQL(null).append(" WHERE CpasType = ").appendValue(_ss.getLSID()); - return (_MaterializedQueryHelper) new _MaterializedQueryHelper.Builder(_ss.getLSID(), "", getExpSchema().getDbSchema().getScope(), viewSql) - .addIndex("CREATE UNIQUE INDEX uq_${NAME}_rowid ON temp.${NAME} (rowid)") - .addIndex("CREATE UNIQUE INDEX uq_${NAME}_lsid ON temp.${NAME} (lsid)") - .addIndex("CREATE INDEX idx_${NAME}_container ON temp.${NAME} (container)") - .addIndex("CREATE INDEX idx_${NAME}_root ON temp.${NAME} (rootmaterialrowid)") - .addInvalidCheck(() -> String.valueOf(getInvalidateCounters(_ss.getLSID()).update.get())) - .build(); - }); - return new SQLFragment("SELECT * FROM ").append(mqh.getFromSql("_cached_view_")); - } - - - /** - * MaterializedQueryHelper has a built-in mechanism for tracking when a temp table needs to be recomputed. - * It does not help with incremental updates (except for providing the upsert() method). - * _MaterializedQueryHelper and _Materialized copy the pattern using class Invalidator. - */ - static class _MaterializedQueryHelper extends MaterializedQueryHelper - { - final String _lsid; - - static class Builder extends MaterializedQueryHelper.Builder - { - String _lsid; - - public Builder(String lsid, String prefix, DbScope scope, SQLFragment select) - { - super(prefix, scope, select); - this._lsid = lsid; - } - - @Override - public _MaterializedQueryHelper build() - { - return new _MaterializedQueryHelper(_lsid, _prefix, _scope, _select, _uptodate, _supplier, _indexes, _max, _isSelectInto); - } - } - - _MaterializedQueryHelper(String lsid, String prefix, DbScope scope, SQLFragment select, @Nullable SQLFragment uptodate, Supplier supplier, @Nullable Collection indexes, long maxTimeToCache, - boolean isSelectIntoSql) - { - super(prefix, scope, select, uptodate, supplier, indexes, maxTimeToCache, isSelectIntoSql); - this._lsid = lsid; - } - - @Override - protected Materialized createMaterialized(String txCacheKey) - { - DbSchema temp = DbSchema.getTemp(); - String name = _prefix + "_" + GUID.makeHash(); - _Materialized materialized = new _Materialized(this, name, txCacheKey, HeartBeat.currentTimeMillis(), "\"" + temp.getName() + "\".\"" + name + "\""); - initMaterialized(materialized); - return materialized; - } - - @Override - protected void incrementalUpdateBeforeSelect(Materialized m) - { - _Materialized materialized = (_Materialized) m; - - boolean lockAcquired = false; - try - { - lockAcquired = materialized.getLock().tryLock(1, TimeUnit.MINUTES); - if (Materialized.LoadingState.ERROR == materialized._loadingState.get()) - throw materialized._loadException; - - if (!materialized.incrementalDeleteCheck.stillValid(0)) - executeIncrementalDelete(); - if (!materialized.incrementalRollupCheck.stillValid(0)) - executeIncrementalRollup(); - if (!materialized.incrementalInsertCheck.stillValid(0)) - executeIncrementalInsert(); - } - catch (RuntimeException|InterruptedException ex) - { - RuntimeException rex = UnexpectedException.wrap(ex); - materialized.setError(rex); - // The only time I'd expect an error is due to a schema change race-condition, but that can happen in any code path. - - // Ensure that next refresh starts clean - _materializedQueries.remove(_lsid); - getInvalidateCounters(_lsid).update.incrementAndGet(); - throw rex; - } - finally - { - if (lockAcquired) - materialized.getLock().unlock(); - } - } - - void upsertWithRetry(SQLFragment sql) - { - // not actually read-only, but we don't want to start an explicit transaction - _scope.executeWithRetryReadOnly((tx) -> upsert(sql)); - } - - void executeIncrementalInsert() - { - SQLFragment incremental = new SQLFragment("INSERT INTO temp.${NAME}\n") - .append("SELECT * FROM (") - .append(getViewSourceSql()).append(") viewsource_\n") - .append("WHERE rowid > (SELECT COALESCE(MAX(rowid),0) FROM temp.${NAME})"); - upsertWithRetry(incremental); - } - - void executeIncrementalDelete() - { - var d = CoreSchema.getInstance().getSchema().getSqlDialect(); - // POSTGRES bug??? the obvious query is _very_ slow O(n^2) - // DELETE FROM temp.${NAME} WHERE rowid NOT IN (SELECT rowid FROM exp.material WHERE cpastype = <<_lsid>>) - SQLFragment incremental = new SQLFragment() - .append("WITH deleted AS (SELECT rowid FROM temp.${NAME} EXCEPT SELECT rowid FROM exp.material WHERE cpastype = ").appendValue(_lsid,d).append(")\n") - .append("DELETE FROM temp.${NAME} WHERE rowid IN (SELECT rowid from deleted)\n"); - upsertWithRetry(incremental); - } - - void executeIncrementalRollup() - { - var d = CoreSchema.getInstance().getSchema().getSqlDialect(); - SQLFragment incremental = new SQLFragment(); - if (d.isPostgreSQL()) - { - incremental - .append("UPDATE temp.${NAME} AS st\n") - .append("SET aliquotcount = expm.aliquotcount, availablealiquotcount = expm.availablealiquotcount, aliquotvolume = expm.aliquotvolume, availablealiquotvolume = expm.availablealiquotvolume, aliquotunit = expm.aliquotunit\n") - .append("FROM exp.Material AS expm\n") - .append("WHERE expm.rowid = st.rowid AND expm.cpastype = ").appendValue(_lsid,d).append(" AND (\n") - .append(" st.aliquotcount IS DISTINCT FROM expm.aliquotcount OR ") - .append(" st.availablealiquotcount IS DISTINCT FROM expm.availablealiquotcount OR ") - .append(" st.aliquotvolume IS DISTINCT FROM expm.aliquotvolume OR ") - .append(" st.availablealiquotvolume IS DISTINCT FROM expm.availablealiquotvolume OR ") - .append(" st.aliquotunit IS DISTINCT FROM expm.aliquotunit") - .append(")"); - } - else - { - // SQL Server 2022 supports IS DISTINCT FROM - incremental - .append("UPDATE st\n") - .append("SET aliquotcount = expm.aliquotcount, availablealiquotcount = expm.availablealiquotcount, aliquotvolume = expm.aliquotvolume, availablealiquotvolume = expm.availablealiquotvolume, aliquotunit = expm.aliquotunit\n") - .append("FROM temp.${NAME} st, exp.Material expm\n") - .append("WHERE expm.rowid = st.rowid AND expm.cpastype = ").appendValue(_lsid,d).append(" AND (\n") - .append(" COALESCE(st.aliquotcount,-2147483648) <> COALESCE(expm.aliquotcount,-2147483648) OR ") - .append(" COALESCE(st.availablealiquotcount,-2147483648) <> COALESCE(expm.availablealiquotcount,-2147483648) OR ") - .append(" COALESCE(st.aliquotvolume,-2147483648) <> COALESCE(expm.aliquotvolume,-2147483648) OR ") - .append(" COALESCE(st.availablealiquotvolume,-2147483648) <> COALESCE(expm.availablealiquotvolume,-2147483648) OR ") - .append(" COALESCE(st.aliquotunit,'-') <> COALESCE(expm.aliquotunit,'-')") - .append(")"); - } - upsertWithRetry(incremental); - } - } - - static class _Materialized extends MaterializedQueryHelper.Materialized - { - final MaterializedQueryHelper.Invalidator incrementalInsertCheck; - final MaterializedQueryHelper.Invalidator incrementalRollupCheck; - final MaterializedQueryHelper.Invalidator incrementalDeleteCheck; - - _Materialized(_MaterializedQueryHelper mqh, String tableName, String cacheKey, long created, String sql) - { - super(mqh, tableName, cacheKey, created, sql); - final InvalidationCounters counters = getInvalidateCounters(mqh._lsid); - incrementalInsertCheck = new MaterializedQueryHelper.SupplierInvalidator(() -> String.valueOf(counters.insert.get())); - incrementalRollupCheck = new MaterializedQueryHelper.SupplierInvalidator(() -> String.valueOf(counters.rollup.get())); - incrementalDeleteCheck = new MaterializedQueryHelper.SupplierInvalidator(() -> String.valueOf(counters.delete.get())); - } - - @Override - public void reset() - { - super.reset(); - long now = HeartBeat.currentTimeMillis(); - incrementalInsertCheck.stillValid(now); - incrementalRollupCheck.stillValid(now); - incrementalDeleteCheck.stillValid(now); - } - - Lock getLock() - { - return _loadingLock; - } - } - - - /* SELECT and JOIN, does not include WHERE */ - private SQLFragment getJoinSQL(Set selectedColumns) - { - TableInfo provisioned = null == _ss ? null : _ss.getTinfo(); - Set provisionedCols = new CaseInsensitiveHashSet(provisioned != null ? provisioned.getColumnNameSet() : Collections.emptySet()); - provisionedCols.remove(Column.RowId.name()); - provisionedCols.remove(Column.LSID.name()); - provisionedCols.remove(Column.Name.name()); - boolean hasProvisionedColumns = containsProvisionedColumns(selectedColumns, provisionedCols); - - boolean hasSampleColumns = false; - boolean hasAliquotColumns = false; - - Set materialCols = new CaseInsensitiveHashSet(_rootTable.getColumnNameSet()); - selectedColumns = computeInnerSelectedColumns(selectedColumns); - - SQLFragment sql = new SQLFragment(); - sql.appendComment("", getSqlDialect()); - sql.append("SELECT "); - String comma = ""; - for (String materialCol : materialCols) - { - // don't need to generate SQL for columns that aren't selected - if (ALL_COLUMNS == selectedColumns || selectedColumns.contains(new FieldKey(null, materialCol))) - { - sql.append(comma).append("m.").appendIdentifier(materialCol); - comma = ", "; - } - } - if (null != provisioned && hasProvisionedColumns) - { - for (ColumnInfo propertyColumn : provisioned.getColumns()) - { - // don't select twice - if ( - Column.RowId.name().equalsIgnoreCase(propertyColumn.getColumnName()) || - Column.LSID.name().equalsIgnoreCase(propertyColumn.getColumnName()) || - Column.Name.name().equalsIgnoreCase(propertyColumn.getColumnName()) - ) - { - continue; - } - - // don't need to generate SQL for columns that aren't selected - if (ALL_COLUMNS == selectedColumns || selectedColumns.contains(propertyColumn.getFieldKey()) || propertyColumn.isMvIndicatorColumn()) - { - sql.append(comma); - boolean rootField = StringUtils.isEmpty(propertyColumn.getDerivationDataScope()) - || ExpSchema.DerivationDataScopeType.ParentOnly.name().equalsIgnoreCase(propertyColumn.getDerivationDataScope()); - if ("genid".equalsIgnoreCase(propertyColumn.getColumnName()) || propertyColumn.isUniqueIdField()) - { - sql.append(propertyColumn.getValueSql("m_aliquot")).append(" AS ").appendIdentifier(propertyColumn.getSelectIdentifier()); - hasAliquotColumns = true; - } - else if (rootField) - { - sql.append(propertyColumn.getValueSql("m_sample")).append(" AS ").appendIdentifier(propertyColumn.getSelectIdentifier()); - hasSampleColumns = true; - } - else - { - sql.append(propertyColumn.getValueSql("m_aliquot")).append(" AS ").appendIdentifier(propertyColumn.getSelectIdentifier()); - hasAliquotColumns = true; - } - comma = ", "; - } - } - } - - sql.append("\nFROM "); - sql.append(_rootTable, "m"); - if (hasSampleColumns) - sql.append(" INNER JOIN ").append(provisioned, "m_sample").append(" ON m.RootMaterialRowId = m_sample.RowId"); - if (hasAliquotColumns) - sql.append(" INNER JOIN ").append(provisioned, "m_aliquot").append(" ON m.RowId = m_aliquot.RowId"); - - sql.appendComment("", getSqlDialect()); - return sql; - } - - private class IdColumnRendererFactory implements DisplayColumnFactory - { - @Override - public DisplayColumn createRenderer(ColumnInfo colInfo) - { - return new IdColumnRenderer(colInfo); - } - } - - private static class IdColumnRenderer extends DataColumn - { - public IdColumnRenderer(ColumnInfo col) - { - super(col); - } - - @Override - protected boolean isDisabledInput(RenderContext ctx) - { - return !super.isDisabledInput() && ctx.getMode() != DataRegion.MODE_INSERT; - } - } - - private static class SampleTypeAmountDisplayColumn extends ExprColumn - { - public SampleTypeAmountDisplayColumn(TableInfo parent, String amountFieldName, String unitFieldName, String label, Set importAliases, Unit typeUnit) - { - super(parent, FieldKey.fromParts(amountFieldName), new SQLFragment( - "(CASE WHEN ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(unitFieldName) - .append(" = ? AND ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(amountFieldName) - .append(" IS NOT NULL THEN ROUND(CAST(").append(ExprColumn.STR_TABLE_ALIAS + ".").append(amountFieldName) - .append(" / ? AS ") - .append(parent.getSqlDialect().isPostgreSQL() ? "DECIMAL" : "DOUBLE PRECISION") - .append("), ?) ELSE ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(amountFieldName) - .append(" END)") - .add(typeUnit.getBase().toString()) - .add(typeUnit.getValue()) - .add(typeUnit.getPrecisionScale()), - JdbcType.DOUBLE); - - setLabel(label); - setImportAliasesSet(importAliases); - } - } - - private static class SampleTypeUnitDisplayColumn extends ExprColumn - { - public SampleTypeUnitDisplayColumn(TableInfo parent, String unitFieldName, Unit typeUnit) - { - super(parent, FieldKey.fromParts(Column.Units.name()), new SQLFragment( - "(CASE WHEN ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(unitFieldName) - .append(" = ? THEN ? ELSE ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(unitFieldName) - .append(" END)") - .add(typeUnit.getBase().toString()) - .add(typeUnit.toString()), - JdbcType.VARCHAR); - } - } - - @Override - public QueryUpdateService getUpdateService() - { - return new SampleTypeUpdateServiceDI(this, _ss); - } - - @Override - public boolean hasPermission(@NotNull UserPrincipal user, @NotNull Class perm) - { - if (_ss == null) - { - // Allow read and delete for exp.Materials. - // Don't allow insert/update on exp.Materials without a sample type. - if (perm == DeletePermission.class || perm == ReadPermission.class) - return getContainer().hasPermission(user, perm); - return false; - } - - if (_ss.isMedia() && perm == ReadPermission.class) - return getContainer().hasPermission(user, MediaReadPermission.class); - - return super.hasPermission(user, perm); - } - - @NotNull - @Override - public List getUniqueIndices() - { - // Rewrite the "idx_material_ak" unique index over "Folder", "SampleSet", "Name" to just "Name" - // Issue 25397: Don't include the "idx_material_ak" index if the "Name" column hasn't been added to the table. - // Some FKs to ExpMaterialTable don't include the "Name" column (e.g. NabBaseTable.Specimen) - String indexName = "idx_material_ak"; - List ret = new ArrayList<>(super.getUniqueIndices()); - if (getColumn("Name") != null) - ret.add(new IndexDefinition(indexName, IndexType.Unique, Arrays.asList(getColumn("Name")), null)); - else - ret.removeIf( def -> def.name().equals(indexName)); - return Collections.unmodifiableList(ret); - } - - - // - // UpdatableTableInfo - // - - - @Override - public @Nullable Long getOwnerObjectId() - { - return OntologyManager.ensureObject(_ss.getContainer(), _ss.getLSID(), (Long) null); - } - - @Nullable - @Override - public CaseInsensitiveHashMap remapSchemaColumns() - { - CaseInsensitiveHashMap m = new CaseInsensitiveHashMap<>(); - - if (null != getRealTable().getColumn("container") && null != getColumn("folder")) - { - m.put("container", "folder"); - } - - for (ColumnInfo col : getColumns()) - { - if (col.getMvColumnName() != null) - m.put(col.getName() + "_" + MvColumn.MV_INDICATOR_SUFFIX, col.getMvColumnName().getName()); - } - - return m; - } - - @Override - public Set getAltMergeKeys(DataIteratorContext context) - { - if (context.getInsertOption().updateOnly && context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate)) - return getAltKeysForUpdate(); - - return MATERIAL_ALT_MERGE_KEYS; - } - - @NotNull - @Override - public Set getAltKeysForUpdate() - { - return MATERIAL_ALT_UPDATE_KEYS; - } - - @Override - @NotNull - public List> getAdditionalRequiredInsertColumns() - { - if (getSampleType() == null) - return Collections.emptyList(); - - try - { - return getRequiredParentImportFields(getSampleType().getRequiredImportAliases()); - } - catch (IOException e) - { - return Collections.emptyList(); - } - } - - @Override - public DataIteratorBuilder persistRows(DataIteratorBuilder data, DataIteratorContext context) - { - TableInfo propertiesTable = _ss.getTinfo(); - - // The specimens sample type doesn't have a properties table - if (propertiesTable == null) - { - return data; - } - - long sampleTypeObjectId = requireNonNull(getOwnerObjectId()); - - // TODO: subclass PersistDataIteratorBuilder to index Materials! not DataClass! - try - { - var persist = new ExpDataIterators.PersistDataIteratorBuilder(data, this, propertiesTable, _ss, getUserSchema().getContainer(), getUserSchema().getUser(), _ss.getImportAliases(), sampleTypeObjectId) - .setFileLinkDirectory(SAMPLETYPE_FILE_DIRECTORY); - ExperimentServiceImpl experimentServiceImpl = ExperimentServiceImpl.get(); - SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(getContainer(), SearchService.PRIORITY.modified); - - persist.setIndexFunction(searchIndexDataKeys -> propertiesTable.getSchema().getScope().addCommitTask(() -> - { - List lsids = searchIndexDataKeys.lsids(); - List orderedRowIds = searchIndexDataKeys.orderedRowIds(); - - // Issue 51263: order by RowId to reduce deadlock - ListUtils.partition(orderedRowIds, 100).forEach(sublist -> - queue.addRunnable((q) -> - { - for (ExpMaterialImpl expMaterial : experimentServiceImpl.getExpMaterials(sublist)) - expMaterial.index(q, this); - }) - ); - - ListUtils.partition(lsids, 100).forEach(sublist -> - queue.addRunnable((q) -> - { - for (ExpMaterialImpl expMaterial : experimentServiceImpl.getExpMaterialsByLsid(sublist)) - expMaterial.index(q, this); - }) - ); - }, DbScope.CommitTaskOption.POSTCOMMIT) - ); - - DataIteratorBuilder builder = LoggingDataIterator.wrap(persist); - return LoggingDataIterator.wrap(new AliasDataIteratorBuilder(builder, getUserSchema().getContainer(), getUserSchema().getUser(), ExperimentService.get().getTinfoMaterialAliasMap(), _ss, true)); - } - catch (IOException e) - { - throw new UncheckedIOException(e); - } - } - - @Override - @NotNull - public AuditBehaviorType getDefaultAuditBehavior() - { - return AuditBehaviorType.DETAILED; - } - - static final Set excludeFromDetailedAuditField; - static - { - var set = new CaseInsensitiveHashSet(); - set.addAll(TableInfo.defaultExcludedDetailedUpdateAuditFields); - set.addAll(ExpDataIterators.NOT_FOR_UPDATE); - // We don't want the inventory columns to show up in the sample timeline audit record; - // they are captured in their own audit record. - set.addAll(InventoryService.InventoryStatusColumn.names()); - excludeFromDetailedAuditField = Collections.unmodifiableSet(set); - } - - @Override - public @NotNull Set getExcludedDetailedUpdateAuditFields() - { - // uniqueId fields don't change in reality, so exclude them from the audit updates - Set excluded = new CaseInsensitiveHashSet(); - excluded.addAll(this.getUniqueIdFields()); - excluded.addAll(excludeFromDetailedAuditField); - return excluded; - } - - @Override - public List> getImportTemplates(ViewContext ctx) - { - // respect any metadata overrides - if (getRawImportTemplates() != null) - return super.getImportTemplates(ctx); - - List> templates = new ArrayList<>(); - ActionURL url = PageFlowUtil.urlProvider(QueryUrls.class).urlCreateExcelTemplate(ctx.getContainer(), getPublicSchemaName(), getName()); - url.addParameter("headerType", ColumnHeaderType.ImportField.name()); - try - { - if (getSampleType() != null && !getSampleType().getImportAliases().isEmpty()) - { - for (String aliasKey : getSampleType().getImportAliases().keySet()) - url.addParameter("includeColumn", aliasKey); - } - } - catch (IOException e) - {} - templates.add(Pair.of("Download Template", url.toString())); - return templates; - } - - @Override - public void overlayMetadata(String tableName, UserSchema schema, Collection errors) - { - if (SamplesSchema.SCHEMA_NAME.equals(schema.getName())) - { - Collection metadata = QueryService.get().findMetadataOverride(schema, SamplesSchema.SCHEMA_METADATA_NAME, false, false, errors, null); - if (null != metadata) - { - overlayMetadata(metadata, schema, errors); - } - } - super.overlayMetadata(tableName, schema, errors); - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.experiment.api; + +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.math3.util.Precision; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.assay.plate.AssayPlateMetadataService; +import org.labkey.api.audit.AuditHandler; +import org.labkey.api.cache.BlockingCache; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.compliance.TableRules; +import org.labkey.api.compliance.TableRulesManager; +import org.labkey.api.data.ColumnHeaderType; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DataColumn; +import org.labkey.api.data.DataRegion; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.DisplayColumn; +import org.labkey.api.data.DisplayColumnFactory; +import org.labkey.api.data.ForeignKey; +import org.labkey.api.data.ImportAliasable; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.MaterializedQueryHelper; +import org.labkey.api.data.MutableColumnInfo; +import org.labkey.api.data.PHI; +import org.labkey.api.data.RenderContext; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.Sort; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.UnionContainerFilter; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.LoggingDataIterator; +import org.labkey.api.dataiterator.SimpleTranslator; +import org.labkey.api.exp.MvColumn; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyColumn; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.ExperimentUrls; +import org.labkey.api.exp.api.NameExpressionOptionService; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.DomainUtil; +import org.labkey.api.exp.query.ExpDataTable; +import org.labkey.api.exp.query.ExpMaterialTable; +import org.labkey.api.exp.query.ExpSampleTypeTable; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.exp.query.SamplesSchema; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.inventory.InventoryService; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; +import org.labkey.api.qc.SampleStatusService; +import org.labkey.api.query.AliasedColumn; +import org.labkey.api.query.DetailsURL; +import org.labkey.api.query.ExprColumn; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.LookupForeignKey; +import org.labkey.api.query.QueryException; +import org.labkey.api.query.QueryForeignKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateService; +import org.labkey.api.query.QueryUrls; +import org.labkey.api.query.RowIdForeignKey; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.column.BuiltInColumnTypes; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.UserPrincipal; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.InsertPermission; +import org.labkey.api.security.permissions.MediaReadPermission; +import org.labkey.api.security.permissions.MoveEntitiesPermission; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.util.GUID; +import org.labkey.api.util.HeartBeat; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StringExpression; +import org.labkey.api.util.UnexpectedException; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.ViewContext; +import org.labkey.data.xml.TableType; +import org.labkey.experiment.ExpDataIterators; +import org.labkey.experiment.ExpDataIterators.AliasDataIteratorBuilder; +import org.labkey.experiment.controllers.exp.ExperimentController; +import org.labkey.experiment.lineage.LineageMethod; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static java.util.Objects.requireNonNull; +import static org.labkey.api.exp.api.SampleTypeDomainKind.ALIQUOT_COUNT_LABEL; +import static org.labkey.api.exp.api.SampleTypeDomainKind.ALIQUOT_VOLUME_LABEL; +import static org.labkey.api.exp.api.SampleTypeDomainKind.AVAILABLE_ALIQUOT_COUNT_LABEL; +import static org.labkey.api.exp.api.SampleTypeDomainKind.AVAILABLE_ALIQUOT_VOLUME_LABEL; +import static org.labkey.api.exp.api.SampleTypeDomainKind.SAMPLETYPE_FILE_DIRECTORY; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotCount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotVolume; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.AvailableAliquotCount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.AvailableAliquotVolume; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; +import static org.labkey.api.util.StringExpressionFactory.AbstractStringExpression.NullValueBehavior.NullResult; +import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.schema; + +public class ExpMaterialTableImpl extends ExpRunItemTableImpl implements ExpMaterialTable +{ + ExpSampleTypeImpl _ss; + Set _uniqueIdFields; + boolean _supportTableRules = true; + + public static final Set MATERIAL_ALT_MERGE_KEYS; + public static final Set MATERIAL_ALT_UPDATE_KEYS; + static { + MATERIAL_ALT_MERGE_KEYS = Set.of(Column.MaterialSourceId.name(), Column.Name.name()); + MATERIAL_ALT_UPDATE_KEYS = Set.of(Column.LSID.name()); + } + + public ExpMaterialTableImpl(UserSchema schema, ContainerFilter cf, @Nullable ExpSampleType sampleType) + { + super(ExpSchema.TableType.Materials.name(), ExperimentServiceImpl.get().getTinfoMaterial(), schema, cf); + setDetailsURL(new DetailsURL(new ActionURL(ExperimentController.ShowMaterialAction.class, schema.getContainer()), Collections.singletonMap("rowId", "rowId"), NullResult)); + setPublicSchemaName(ExpSchema.SCHEMA_NAME); + addAllowablePermission(InsertPermission.class); + addAllowablePermission(UpdatePermission.class); + addAllowablePermission(MoveEntitiesPermission.class); + setAllowedInsertOption(QueryUpdateService.InsertOption.MERGE); + setSampleType(sampleType); + } + + public Set getUniqueIdFields() + { + if (_uniqueIdFields == null) + { + _uniqueIdFields = new CaseInsensitiveHashSet(); + _uniqueIdFields.addAll(getColumns().stream().filter(ColumnInfo::isUniqueIdField).map(ColumnInfo::getName).collect(Collectors.toSet())); + } + return _uniqueIdFields; + } + + @Override + protected ColumnInfo resolveColumn(String name) + { + ColumnInfo result = super.resolveColumn(name); + if (result == null) + { + if ("CpasType".equalsIgnoreCase(name)) + result = createColumn(Column.SampleSet.name(), Column.SampleSet); + else if (Column.Property.name().equalsIgnoreCase(name)) + result = createPropertyColumn(Column.Property.name()); + else if (Column.QueryableInputs.name().equalsIgnoreCase(name)) + result = createColumn(Column.QueryableInputs.name(), Column.QueryableInputs); + } + return result; + } + + @Override + public ColumnInfo getExpObjectColumn() + { + var ret = wrapColumn("ExpMaterialTableImpl_object_", _rootTable.getColumn("objectid")); + ret.setConceptURI(BuiltInColumnTypes.EXPOBJECTID_CONCEPT_URI); + return ret; + } + + @Override + public AuditHandler getAuditHandler(AuditBehaviorType auditBehaviorType) + { + if (getUserSchema().getName().equalsIgnoreCase(SamplesSchema.SCHEMA_NAME)) + { + // Special case sample auditing to help build a useful timeline view + return SampleTypeServiceImpl.get(); + } + + return super.getAuditHandler(auditBehaviorType); + } + + @Override + public MutableColumnInfo createColumn(String alias, Column column) + { + switch (column) + { + case Folder -> + { + return wrapColumn(alias, _rootTable.getColumn("Container")); + } + case LSID -> + { + return wrapColumn(alias, _rootTable.getColumn(Column.LSID.name())); + } + case MaterialSourceId -> + { + var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.MaterialSourceId.name())); + columnInfo.setFk(new LookupForeignKey(getLookupContainerFilter(), null, null, null, null, "RowId", "Name") + { + @Override + public TableInfo getLookupTableInfo() + { + ExpSampleTypeTable sampleTypeTable = ExperimentService.get().createSampleTypeTable(ExpSchema.TableType.SampleSets.toString(), _userSchema, getLookupContainerFilter()); + sampleTypeTable.populate(); + return sampleTypeTable; + } + + @Override + public StringExpression getURL(ColumnInfo parent) + { + return super.getURL(parent, true); + } + }); + columnInfo.setUserEditable(false); + columnInfo.setReadOnly(true); + columnInfo.setHidden(true); + return columnInfo; + } + case RootMaterialRowId -> + { + var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.RootMaterialRowId.name())); + columnInfo.setFk(getExpSchema().getMaterialForeignKey(getLookupContainerFilter(), Column.RowId.name())); + columnInfo.setLabel("Root Material"); + columnInfo.setUserEditable(false); + + // NK: Here we mark the column as not required AND nullable which is the opposite of the database where + // a NOT NULL constraint is in place. This is done to avoid the RequiredValidator check upon updating a row. + // See ExpMaterialValidatorIterator. + columnInfo.setRequired(false); + columnInfo.setNullable(true); + + return columnInfo; + } + case AliquotedFromLSID -> + { + var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.AliquotedFromLSID.name())); + columnInfo.setSqlTypeName("lsidtype"); + columnInfo.setFk(getExpSchema().getMaterialForeignKey(getLookupContainerFilter(), Column.LSID.name())); + columnInfo.setLabel("Aliquoted From Parent"); + return columnInfo; + } + case IsAliquot -> + { + String rootMaterialRowIdField = ExprColumn.STR_TABLE_ALIAS + "." + Column.RootMaterialRowId.name(); + String rowIdField = ExprColumn.STR_TABLE_ALIAS + "." + Column.RowId.name(); + ExprColumn columnInfo = new ExprColumn(this, FieldKey.fromParts(Column.IsAliquot.name()), new SQLFragment( + "(CASE WHEN (" + rootMaterialRowIdField + " = " + rowIdField + ") THEN ").append(getSqlDialect().getBooleanFALSE()) + .append(" WHEN ").append(rowIdField).append(" IS NOT NULL THEN ").append(getSqlDialect().getBooleanTRUE()) // Issue 52745 + .append(" ELSE NULL END)"), JdbcType.BOOLEAN); + columnInfo.setLabel("Is Aliquot"); + columnInfo.setDescription("Identifies if the material is a sample or an aliquot"); + columnInfo.setUserEditable(false); + columnInfo.setReadOnly(true); + columnInfo.setHidden(false); + return columnInfo; + } + case Name -> + { + var nameCol = wrapColumn(alias, _rootTable.getColumn(column.toString())); + // shut off this field in insert and update views if user specified names are not allowed + if (!NameExpressionOptionService.get().getAllowUserSpecificNamesValue(getContainer())) + { + nameCol.setShownInInsertView(false); + nameCol.setShownInUpdateView(false); + } + return nameCol; + } + case RawAmount -> + { + var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.StoredAmount.name())); + columnInfo.setDisplayColumnFactory(colInfo -> new SampleTypeAmountPrecisionDisplayColumn(colInfo, null)); + columnInfo.setDescription("The amount of this sample, in the base unit for the sample type's display unit (if defined), currently on hand."); + columnInfo.setUserEditable(false); + columnInfo.setReadOnly(true); + return columnInfo; + } + case StoredAmount -> + { + String label = StoredAmount.label(); + Set importAliases = Set.of(label, "Stored Amount"); + Unit typeUnit = getSampleTypeUnit(); + if (typeUnit != null) + { + SampleTypeAmountDisplayColumn columnInfo = new SampleTypeAmountDisplayColumn(this, Column.StoredAmount.name(), Column.Units.name(), label, importAliases, typeUnit); + columnInfo.setDisplayColumnFactory(colInfo -> new SampleTypeAmountPrecisionDisplayColumn(colInfo, typeUnit)); + columnInfo.setDescription("The amount of this sample, in the display unit for the sample type, currently on hand."); + columnInfo.setShownInUpdateView(true); + columnInfo.setShownInInsertView(true); + columnInfo.setUserEditable(true); + columnInfo.setCalculated(false); + return columnInfo; + } + else + { + var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.StoredAmount.name())); + columnInfo.setDisplayColumnFactory(colInfo -> new SampleTypeAmountPrecisionDisplayColumn(colInfo, null)); + columnInfo.setLabel(label); + columnInfo.setImportAliasesSet(importAliases); + columnInfo.setDescription("The amount of this sample currently on hand."); + return columnInfo; + } + } + case RawUnits -> + { + var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.Units.name())); + columnInfo.setDescription("The units associated with the Stored Amount for this sample."); + columnInfo.setUserEditable(false); + columnInfo.setReadOnly(true); + return columnInfo; + } + case Units -> + { + ForeignKey fk = new LookupForeignKey("Value", "Value") + { + @Override + public @Nullable TableInfo getLookupTableInfo() + { + return getExpSchema().getTable(ExpSchema.MEASUREMENT_UNITS_TABLE); + } + }; + + Unit typeUnit = getSampleTypeUnit(); + if (typeUnit != null) + { + SampleTypeUnitDisplayColumn columnInfo = new SampleTypeUnitDisplayColumn(this, Column.Units.name(), typeUnit); + columnInfo.setFk(fk); + columnInfo.setDescription("The sample type display units associated with the Amount for this sample."); + columnInfo.setShownInUpdateView(true); + columnInfo.setShownInInsertView(true); + columnInfo.setUserEditable(true); + columnInfo.setCalculated(false); + return columnInfo; + } + else + { + var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.Units.name())); + columnInfo.setFk(fk); + columnInfo.setDescription("The units associated with the Stored Amount for this sample."); + return columnInfo; + } + } + case Description -> + { + return wrapColumn(alias, _rootTable.getColumn(Column.Description.name())); + } + case SampleSet -> + { + var columnInfo = wrapColumn(alias, _rootTable.getColumn("CpasType")); + // NOTE: populateColumns() overwrites this with a QueryForeignKey. Can this be removed? + columnInfo.setFk(new LookupForeignKey(getContainerFilter(), null, null, null, null, "LSID", "Name") + { + @Override + public TableInfo getLookupTableInfo() + { + ExpSampleTypeTable sampleTypeTable = ExperimentService.get().createSampleTypeTable(ExpSchema.TableType.SampleSets.toString(), _userSchema, getLookupContainerFilter()); + sampleTypeTable.populate(); + return sampleTypeTable; + } + + @Override + public StringExpression getURL(ColumnInfo parent) + { + return super.getURL(parent, true); + } + }); + return columnInfo; + } + case SourceProtocolLSID -> + { + // NOTE: This column is incorrectly named "Protocol", but we are keeping it for backwards compatibility to avoid breaking queries in hvtnFlow module + ExprColumn columnInfo = new ExprColumn(this, ExpDataTable.Column.Protocol.toString(), new SQLFragment( + "(SELECT ProtocolLSID FROM " + ExperimentServiceImpl.get().getTinfoProtocolApplication() + " pa " + + " WHERE pa.RowId = " + ExprColumn.STR_TABLE_ALIAS + ".SourceApplicationId)"), JdbcType.VARCHAR); + columnInfo.setSqlTypeName("lsidtype"); + columnInfo.setFk(getExpSchema().getProtocolForeignKey(getContainerFilter(), "LSID")); + columnInfo.setLabel("Source Protocol"); + columnInfo.setDescription("Contains a reference to the protocol for the protocol application that created this sample"); + columnInfo.setUserEditable(false); + columnInfo.setReadOnly(true); + columnInfo.setHidden(true); + return columnInfo; + } + case SourceProtocolApplication -> + { + var columnInfo = wrapColumn(alias, _rootTable.getColumn("SourceApplicationId")); + columnInfo.setFk(getExpSchema().getProtocolApplicationForeignKey(getContainerFilter())); + columnInfo.setUserEditable(false); + columnInfo.setReadOnly(true); + columnInfo.setHidden(true); + columnInfo.setAutoIncrement(false); + return columnInfo; + } + case SourceApplicationInput -> + { + var col = createEdgeColumn(alias, Column.SourceProtocolApplication, ExpSchema.TableType.MaterialInputs); + col.setDescription("Contains a reference to the MaterialInput row between this ExpMaterial and it's SourceProtocolApplication"); + col.setHidden(true); + return col; + } + case RunApplication -> + { + SQLFragment sql = new SQLFragment("(SELECT pa.rowId FROM ") + .append(ExperimentService.get().getTinfoProtocolApplication(), "pa") + .append(" WHERE pa.runId = ").append(ExprColumn.STR_TABLE_ALIAS).append(".runId") + .append(" AND pa.cpasType = ").appendValue(ExpProtocol.ApplicationType.ExperimentRunOutput) + .append(")"); + + var col = new ExprColumn(this, alias, sql, JdbcType.INTEGER); + col.setFk(getExpSchema().getProtocolApplicationForeignKey(getContainerFilter())); + col.setDescription("Contains a reference to the ExperimentRunOutput protocol application of the run that created this sample"); + col.setUserEditable(false); + col.setReadOnly(true); + col.setHidden(true); + return col; + } + case RunApplicationOutput -> + { + var col = createEdgeColumn(alias, Column.RunApplication, ExpSchema.TableType.MaterialInputs); + col.setDescription("Contains a reference to the MaterialInput row between this ExpMaterial and it's RunOutputApplication"); + return col; + } + case Run -> + { + var ret = wrapColumn(alias, _rootTable.getColumn("RunId")); + ret.setReadOnly(true); + return ret; + } + case RowId -> + { + var ret = wrapColumn(alias, _rootTable.getColumn("RowId")); + // When no sorts are added by views, QueryServiceImpl.createDefaultSort() adds the primary key's default sort direction + ret.setSortDirection(Sort.SortDirection.DESC); + ret.setFk(new RowIdForeignKey(ret)); + ret.setUserEditable(false); + ret.setHidden(true); + ret.setShownInInsertView(false); + ret.setHasDbSequence(true); + ret.setIsRootDbSequence(true); + return ret; + } + case Property -> + { + return createPropertyColumn(alias); + } + case Flag -> + { + return createFlagColumn(alias); + } + case Created -> + { + return wrapColumn(alias, _rootTable.getColumn("Created")); + } + case CreatedBy -> + { + return createUserColumn(alias, _rootTable.getColumn("CreatedBy")); + } + case Modified -> + { + return wrapColumn(alias, _rootTable.getColumn("Modified")); + } + case ModifiedBy -> + { + return createUserColumn(alias, _rootTable.getColumn("ModifiedBy")); + } + case Alias -> + { + return createAliasColumn(alias, ExperimentService.get()::getTinfoMaterialAliasMap); + } + case Inputs -> + { + return createLineageColumn(this, alias, true, false); + } + case QueryableInputs -> + { + return createLineageColumn(this, alias, true, true); + } + case Outputs -> + { + return createLineageColumn(this, alias, false, false); + } + case Properties -> + { + return createPropertiesColumn(alias); + } + case SampleState -> + { + boolean statusEnabled = SampleStatusService.get().supportsSampleStatus() && !SampleStatusService.get().getAllProjectStates(getContainer()).isEmpty(); + var ret = wrapColumn(alias, _rootTable.getColumn(column.name())); + ret.setLabel("Status"); + ret.setHidden(!statusEnabled); + ret.setShownInDetailsView(statusEnabled); + ret.setShownInInsertView(statusEnabled); + ret.setShownInUpdateView(statusEnabled); + ret.setRemapMissingBehavior(SimpleTranslator.RemapMissingBehavior.Error); + ret.setFk(new QueryForeignKey.Builder(getUserSchema(), getSampleStatusLookupContainerFilter()) + .schema(getExpSchema()).table(ExpSchema.TableType.SampleStatus).display("Label")); + return ret; + } + case AliquotCount -> + { + var ret = wrapColumn(alias, _rootTable.getColumn(AliquotCount.name())); + ret.setLabel(ALIQUOT_COUNT_LABEL); + return ret; + } + case AliquotVolume -> + { + var ret = wrapColumn(alias, _rootTable.getColumn(AliquotVolume.name())); + ret.setLabel(ALIQUOT_VOLUME_LABEL); + return ret; + } + case AvailableAliquotVolume -> + { + var ret = wrapColumn(alias, _rootTable.getColumn(AvailableAliquotVolume.name())); + ret.setLabel(AVAILABLE_ALIQUOT_VOLUME_LABEL); + return ret; + } + case AvailableAliquotCount -> + { + var ret = wrapColumn(alias, _rootTable.getColumn(AvailableAliquotCount.name())); + ret.setLabel(AVAILABLE_ALIQUOT_COUNT_LABEL); + return ret; + } + case AliquotUnit -> + { + var ret = wrapColumn(alias, _rootTable.getColumn("AliquotUnit")); + ret.setShownInDetailsView(false); + return ret; + } + case MaterialExpDate -> + { + var ret = wrapColumn(alias, _rootTable.getColumn("MaterialExpDate")); + ret.setLabel("Expiration Date"); + ret.setShownInDetailsView(true); + ret.setShownInInsertView(true); + ret.setShownInUpdateView(true); + return ret; + } + default -> throw new IllegalArgumentException("Unknown column " + column); + } + } + + @Override + public MutableColumnInfo createPropertyColumn(String alias) + { + var ret = super.createPropertyColumn(alias); + if (_ss != null) + { + final TableInfo t = _ss.getTinfo(); + if (t != null) + { + ret.setFk(new LookupForeignKey() + { + @Override + public TableInfo getLookupTableInfo() + { + return t; + } + + @Override + protected ColumnInfo getPkColumn(TableInfo table) + { + return t.getColumn("lsid"); + } + }); + } + } + ret.setIsUnselectable(true); + ret.setDescription("A holder for any custom fields associated with this sample"); + ret.setHidden(true); + return ret; + } + + private Unit getSampleTypeUnit() + { + Unit typeUnit = null; + if (_ss != null && _ss.getMetricUnit() != null) + typeUnit = Unit.fromName(_ss.getMetricUnit()); + return typeUnit; + } + + private void setSampleType(@Nullable ExpSampleType st) + { + checkLocked(); + if (_ss != null) + { + throw new IllegalStateException("Cannot unset sample type"); + } + if (st != null && !(st instanceof ExpSampleTypeImpl)) + { + throw new IllegalArgumentException("Expected sample type to be an instance of " + ExpSampleTypeImpl.class.getName() + " but was a " + st.getClass().getName()); + } + _ss = (ExpSampleTypeImpl) st; + if (_ss != null) + { + setPublicSchemaName(SamplesSchema.SCHEMA_NAME); + setName(st.getName()); + + String description = _ss.getDescription(); + if (StringUtils.isEmpty(description)) + description = "Contains one row per sample in the " + _ss.getName() + " sample type"; + setDescription(description); + + if (canUserAccessPhi()) + { + ActionURL url = PageFlowUtil.urlProvider(ExperimentUrls.class).getImportSamplesURL(getContainer(), _ss.getName()); + setImportURL(new DetailsURL(url)); + } + } + } + + public ExpSampleType getSampleType() + { + return _ss; + } + + @Override + protected void populateColumns() + { + var st = getSampleType(); + var rowIdCol = addColumn(Column.RowId); + addColumn(Column.MaterialSourceId); + addColumn(Column.SourceProtocolApplication); + addColumn(Column.SourceApplicationInput); + addColumn(Column.RunApplication); + addColumn(Column.RunApplicationOutput); + addColumn(Column.SourceProtocolLSID); + + var nameCol = addColumn(Column.Name); + if (st != null && st.hasNameAsIdCol()) + { + // Show the Name field but don't mark is as required when using name expressions + if (st.hasNameExpression()) + { + var nameExpression = st.getNameExpression(); + nameCol.setNameExpression(nameExpression); + nameCol.setNullable(true); + String nameExpressionPreview = getExpNameExpressionPreview(getUserSchema().getSchemaName(), st.getName(), getUserSchema().getUser()); + String desc = appendNameExpressionDescription(nameCol.getDescription(), nameExpression, nameExpressionPreview); + nameCol.setDescription(desc); + } + else + { + nameCol.setNullable(false); + } + } + else + { + nameCol.setReadOnly(true); + nameCol.setShownInInsertView(false); + } + + addColumn(Column.Alias); + addColumn(Column.Description); + + var typeColumnInfo = addColumn(Column.SampleSet); + typeColumnInfo.setFk(new QueryForeignKey(_userSchema, getContainerFilter(), ExpSchema.SCHEMA_NAME, getContainer(), null, ExpSchema.TableType.SampleSets.name(), "lsid", null) + { + @Override + protected ContainerFilter getLookupContainerFilter() + { + // Be sure that we can resolve the sample type if it's defined in a separate container. + // Same as CurrentPlusProjectAndShared but includes SampleSet's container as well. + // Issue 37982: Sample Type: Link to precursor sample type does not resolve correctly if sample has + // parents in current sample type and a sample type in the parent container + Set containers = new HashSet<>(); + if (null != st) + containers.add(st.getContainer()); + containers.add(getContainer()); + if (getContainer().getProject() != null) + containers.add(getContainer().getProject()); + containers.add(ContainerManager.getSharedContainer()); + ContainerFilter cf = new ContainerFilter.CurrentPlusExtras(_userSchema.getContainer(), _userSchema.getUser(), containers); + + if (null != _containerFilter && _containerFilter.getType() != ContainerFilter.Type.Current) + cf = new UnionContainerFilter(_containerFilter, cf); + return cf; + } + }); + + typeColumnInfo.setReadOnly(true); + typeColumnInfo.setUserEditable(false); + typeColumnInfo.setShownInInsertView(false); + + addColumn(Column.MaterialExpDate); + addContainerColumn(Column.Folder, null); + var runCol = addColumn(Column.Run); + runCol.setFk(new ExpSchema(_userSchema.getUser(), getContainer()).getRunIdForeignKey(getContainerFilter())); + runCol.setShownInInsertView(false); + runCol.setShownInUpdateView(false); + + var colLSID = addColumn(Column.LSID); + colLSID.setHidden(true); + colLSID.setReadOnly(true); + colLSID.setUserEditable(false); + colLSID.setShownInInsertView(false); + colLSID.setShownInDetailsView(false); + colLSID.setShownInUpdateView(false); + + var rootRowId = addColumn(Column.RootMaterialRowId); + rootRowId.setHidden(true); + rootRowId.setReadOnly(true); + rootRowId.setUserEditable(false); + rootRowId.setShownInInsertView(false); + rootRowId.setShownInDetailsView(false); + rootRowId.setShownInUpdateView(false); + + var aliquotParentLSID = addColumn(Column.AliquotedFromLSID); + aliquotParentLSID.setHidden(true); + aliquotParentLSID.setReadOnly(true); + aliquotParentLSID.setUserEditable(false); + aliquotParentLSID.setShownInInsertView(false); + aliquotParentLSID.setShownInDetailsView(false); + aliquotParentLSID.setShownInUpdateView(false); + + addColumn(Column.IsAliquot); + addColumn(Column.Created); + addColumn(Column.CreatedBy); + addColumn(Column.Modified); + addColumn(Column.ModifiedBy); + + List defaultCols = new ArrayList<>(); + defaultCols.add(FieldKey.fromParts(Column.Name)); + defaultCols.add(FieldKey.fromParts(Column.MaterialExpDate)); + boolean hasProductFolders = getContainer().hasProductFolders(); + if (hasProductFolders) + defaultCols.add(FieldKey.fromParts(Column.Folder)); + defaultCols.add(FieldKey.fromParts(Column.Run)); + + if (st == null) + defaultCols.add(FieldKey.fromParts(Column.SampleSet)); + + addColumn(Column.Flag); + + var statusColInfo = addColumn(Column.SampleState); + boolean statusEnabled = SampleStatusService.get().supportsSampleStatus() && !SampleStatusService.get().getAllProjectStates(getContainer()).isEmpty(); + statusColInfo.setShownInDetailsView(statusEnabled); + statusColInfo.setShownInInsertView(statusEnabled); + statusColInfo.setShownInUpdateView(statusEnabled); + statusColInfo.setHidden(!statusEnabled); + statusColInfo.setRemapMissingBehavior(SimpleTranslator.RemapMissingBehavior.Error); + if (statusEnabled) + defaultCols.add(FieldKey.fromParts(Column.SampleState)); + statusColInfo.setFk(new QueryForeignKey.Builder(getUserSchema(), getSampleStatusLookupContainerFilter()) + .schema(getExpSchema()).table(ExpSchema.TableType.SampleStatus).display("Label")); + + // TODO is this a real Domain??? + if (st != null && !"urn:lsid:labkey.com:SampleSource:Default".equals(st.getDomain().getTypeURI())) + { + defaultCols.add(FieldKey.fromParts(Column.Flag)); + addSampleTypeColumns(st, defaultCols); + + setName(_ss.getName()); + + ActionURL gridUrl = new ActionURL(ExperimentController.ShowSampleTypeAction.class, getContainer()); + gridUrl.addParameter("rowId", st.getRowId()); + setGridURL(new DetailsURL(gridUrl)); + } + + List calculatedFieldKeys = DomainUtil.getCalculatedFieldsForDefaultView(this); + defaultCols.addAll(calculatedFieldKeys); + + addColumn(Column.AliquotCount); + addColumn(Column.AliquotVolume); + addColumn(Column.AliquotUnit); + addColumn(Column.AvailableAliquotCount); + addColumn(Column.AvailableAliquotVolume); + + addColumn(Column.StoredAmount); + defaultCols.add(FieldKey.fromParts(Column.StoredAmount)); + + addColumn(Column.Units); + defaultCols.add(FieldKey.fromParts(Column.Units)); + + var rawAmountColumn = addColumn(Column.RawAmount); + rawAmountColumn.setDisplayColumnFactory(new DisplayColumnFactory() + { + @Override + public DisplayColumn createRenderer(ColumnInfo colInfo) + { + return new DataColumn(colInfo) + { + @Override + public void addQueryFieldKeys(Set keys) + { + super.addQueryFieldKeys(keys); + keys.add(FieldKey.fromParts(Column.StoredAmount)); + + } + }; + } + }); + rawAmountColumn.setHidden(true); + rawAmountColumn.setShownInDetailsView(false); + rawAmountColumn.setShownInInsertView(false); + rawAmountColumn.setShownInUpdateView(false); + + var rawUnitsColumn = addColumn(Column.RawUnits); + rawUnitsColumn.setDisplayColumnFactory(new DisplayColumnFactory() + { + @Override + public DisplayColumn createRenderer(ColumnInfo colInfo) + { + return new DataColumn(colInfo) + { + @Override + public void addQueryFieldKeys(Set keys) + { + super.addQueryFieldKeys(keys); + keys.add(FieldKey.fromParts(Column.Units)); + + } + }; + } + }); + rawUnitsColumn.setHidden(true); + rawUnitsColumn.setShownInDetailsView(false); + rawUnitsColumn.setShownInInsertView(false); + rawUnitsColumn.setShownInUpdateView(false); + + if (InventoryService.get() != null && (st == null || !st.isMedia())) + defaultCols.addAll(InventoryService.get().addInventoryStatusColumns(st == null ? null : st.getMetricUnit(), this, getContainer(), _userSchema.getUser())); + + SQLFragment sql; + UserSchema plateUserSchema; + // Issue 53194 : this would be the case for linked to study samples. The contextual role is set up from the study dataset + // for the source sample, we want to allow the plate schema to inherit any contextual roles to allow querying + // against tables in that schema. + if (_userSchema instanceof UserSchema.HasContextualRoles samplesSchema && !samplesSchema.getContextualRoles().isEmpty()) + plateUserSchema = AssayPlateMetadataService.get().getPlateSchema(_userSchema, samplesSchema.getContextualRoles()); + else + plateUserSchema = QueryService.get().getUserSchema(_userSchema.getUser(), _userSchema.getContainer(), "plate"); + + if (plateUserSchema != null && plateUserSchema.getTable("Well") != null) + { + String rowIdField = ExprColumn.STR_TABLE_ALIAS + "." + Column.RowId.name(); + SQLFragment existsSubquery = new SQLFragment() + .append("SELECT 1 FROM ") + .append(plateUserSchema.getTable("Well"), "well") + .append(" WHERE well.sampleid = ").append(rowIdField); + + sql = new SQLFragment() + .append("CASE WHEN EXISTS (") + .append(existsSubquery) + .append(") THEN 'Plated' ") + .append("WHEN ").append(ExprColumn.STR_TABLE_ALIAS).append(".RowId").append(" IS NOT NULL THEN 'Not Plated' ")// Issue 52745 + .append("ELSE NULL END"); + } + else + { + sql = new SQLFragment("(SELECT NULL)"); + } + var col = new ExprColumn(this, Column.IsPlated.name(), sql, JdbcType.VARCHAR); + col.setDescription("Whether the sample that has been plated, if plating is supported."); + col.setUserEditable(false); + col.setReadOnly(true); + col.setShownInDetailsView(false); + col.setShownInInsertView(false); + col.setShownInUpdateView(false); + if (plateUserSchema != null) + col.setURL(DetailsURL.fromString("plate-isPlated.api?sampleId=${" + Column.RowId.name() + "}")); + addColumn(col); + + addVocabularyDomains(); + + addColumn(Column.Properties); + + var colInputs = addColumn(Column.Inputs); + addMethod("Inputs", new LineageMethod(colInputs, true), Set.of(colInputs.getFieldKey())); + + var colOutputs = addColumn(Column.Outputs); + addMethod("Outputs", new LineageMethod(colOutputs, false), Set.of(colOutputs.getFieldKey())); + + addExpObjectMethod(); + + ActionURL detailsUrl = new ActionURL(ExperimentController.ShowMaterialAction.class, getContainer()); + DetailsURL url = new DetailsURL(detailsUrl, Collections.singletonMap("rowId", "RowId"), NullResult); + nameCol.setURL(url); + rowIdCol.setURL(url); + setDetailsURL(url); + + if (canUserAccessPhi()) + { + ActionURL updateActionURL = PageFlowUtil.urlProvider(ExperimentUrls.class).getUpdateMaterialQueryRowAction(getContainer(), this); + setUpdateURL(new DetailsURL(updateActionURL, Collections.singletonMap("RowId", "RowId"))); + + ActionURL insertActionURL = PageFlowUtil.urlProvider(ExperimentUrls.class).getInsertMaterialQueryRowAction(getContainer(), this); + setInsertURL(new DetailsURL(insertActionURL)); + } + else + { + setImportURL(LINK_DISABLER); + setInsertURL(LINK_DISABLER); + setUpdateURL(LINK_DISABLER); + } + + setTitleColumn(Column.Name.toString()); + + setDefaultVisibleColumns(defaultCols); + + MutableColumnInfo lineageLookup = ClosureQueryHelper.createAncestorLookupColumnInfo("Ancestors", this, _rootTable.getColumn("rowid"), _ss, true); + addColumn(lineageLookup); + } + + private ContainerFilter getSampleStatusLookupContainerFilter() + { + // The default lookup container filter is Current, but we want to have the default be CurrentPlusProjectAndShared + // for the sample status lookup since in the app project context we want to share status definitions across + // a given project instead of creating duplicate statuses in each subfolder project. + ContainerFilter.Type type = QueryService.get().getContainerFilterTypeForLookups(getContainer()); + type = type == null ? ContainerFilter.Type.CurrentPlusProjectAndShared : type; + return type.create(getUserSchema()); + } + + @Override + public Domain getDomain() + { + return getDomain(false); + } + + @Override + public Domain getDomain(boolean forUpdate) + { + return _ss == null ? null : _ss.getDomain(forUpdate); + } + + + public static String appendNameExpressionDescription(String currentDescription, String nameExpression, String nameExpressionPreview) + { + if (nameExpression == null) + return currentDescription; + + StringBuilder sb = new StringBuilder(); + if (currentDescription != null && !currentDescription.isEmpty()) + { + sb.append(currentDescription); + if (!currentDescription.endsWith(".")) + sb.append("."); + sb.append("\n"); + } + + sb.append("\nIf not provided, a unique name will be generated from the expression:\n"); + sb.append(nameExpression); + sb.append("."); + if (!StringUtils.isEmpty(nameExpressionPreview)) + { + sb.append("\nExample of name that will be generated from the current pattern: \n"); + sb.append(nameExpressionPreview); + } + + return sb.toString(); + } + + private void addSampleTypeColumns(ExpSampleType st, List visibleColumns) + { + TableInfo dbTable = ((ExpSampleTypeImpl)st).getTinfo(); + if (null == dbTable) + return; + + UserSchema schema = getUserSchema(); + Domain domain = st.getDomain(); + ColumnInfo rowIdColumn = getColumn(Column.RowId); + ColumnInfo lsidColumn = getColumn(Column.LSID); + ColumnInfo nameColumn = getColumn(Column.Name); + + visibleColumns.remove(FieldKey.fromParts(Column.Run.name())); + + // When not using name expressions, mark the ID columns as required. + // NOTE: If not explicitly set, the first domain property will be chosen as the ID column. + final List idCols = st.hasNameExpression() ? Collections.emptyList() : st.getIdCols(); + + Set mvColumns = domain.getProperties().stream() + .filter(ImportAliasable::isMvEnabled) + .map(dp -> FieldKey.fromParts(dp.getPropertyDescriptor().getMvIndicatorStorageColumnName())) + .collect(Collectors.toSet()); + + for (ColumnInfo dbColumn : dbTable.getColumns()) + { + // Don't include PHI columns in full text search index + // CONSIDER: Can we move this to a base class? Maybe in .addColumn() + if (schema.getUser().isSearchUser() && !dbColumn.getPHI().isLevelAllowed(PHI.NotPHI)) + continue; + + if ( + rowIdColumn.getFieldKey().equals(dbColumn.getFieldKey()) || + lsidColumn.getFieldKey().equals(dbColumn.getFieldKey()) || + nameColumn.getFieldKey().equals(dbColumn.getFieldKey()) + ) + { + continue; + } + + var wrapped = wrapColumnFromJoinedTable(dbColumn.getName(), dbColumn); + + // TODO missing values? comments? flags? + DomainProperty dp = domain.getPropertyByURI(dbColumn.getPropertyURI()); + var propColumn = copyColumnFromJoinedTable(null==dp ? dbColumn.getName() : dp.getName(), wrapped); + if (propColumn.getName().equalsIgnoreCase("genid")) + { + propColumn.setHidden(true); + propColumn.setUserEditable(false); + propColumn.setShownInDetailsView(false); + propColumn.setShownInInsertView(false); + propColumn.setShownInUpdateView(false); + } + if (null != dp) + { + PropertyColumn.copyAttributes(schema.getUser(), propColumn, dp.getPropertyDescriptor(), schema.getContainer(), + SchemaKey.fromParts("samples"), st.getName(), FieldKey.fromParts("RowId"), null, getLookupContainerFilter()); + + if (idCols.contains(dp)) + { + propColumn.setNullable(false); + propColumn.setDisplayColumnFactory(new IdColumnRendererFactory()); + } + + // Issue 38341: domain designer advanced settings 'show in default view' setting is not respected + if (!propColumn.isHidden()) + { + visibleColumns.add(propColumn.getFieldKey()); + } + + if (propColumn.isMvEnabled()) + { + // The column in the physical table has a "_MVIndicator" suffix, but we want to expose + // it with a "MVIndicator" suffix (no underscore) + var mvColumn = new AliasedColumn(this, dp.getName() + MvColumn.MV_INDICATOR_SUFFIX, + StorageProvisioner.get().getMvIndicatorColumn(dbTable, dp.getPropertyDescriptor(), "No MV column found for '" + dp.getName() + "' in sample type '" + getName() + "'")); + mvColumn.setLabel(dp.getLabel() != null ? dp.getLabel() : dp.getName() + " MV Indicator"); + mvColumn.setSqlTypeName("VARCHAR"); + mvColumn.setPropertyURI(dp.getPropertyURI()); + mvColumn.setNullable(true); + mvColumn.setUserEditable(false); + mvColumn.setHidden(true); + mvColumn.setMvIndicatorColumn(true); + + addColumn(mvColumn); + propColumn.setMvColumnName(FieldKey.fromParts(dp.getName() + MvColumn.MV_INDICATOR_SUFFIX)); + } + } + + if (!mvColumns.contains(propColumn.getFieldKey())) + addColumn(propColumn); + + } + + setDefaultVisibleColumns(visibleColumns); + } + + // These are mostly fields that are wrapped by fields with different names (see createColumn()) + // we could handle each case separately, but this is easier + static final Set wrappedFieldKeys = Set.of( + new FieldKey(null, "objectid"), + new FieldKey(null, "RowId"), + new FieldKey(null, "LSID"), // Flag + new FieldKey(null, "SourceApplicationId"), // SourceProtocolApplication + new FieldKey(null, "runId"), // Run, RunApplication + new FieldKey(null, "CpasType")); // SampleSet + static final Set ALL_COLUMNS = Set.of(); + + private @NotNull Set computeInnerSelectedColumns(Set selectedColumns) + { + if (null == selectedColumns) + return ALL_COLUMNS; + selectedColumns = new TreeSet<>(selectedColumns); + if (selectedColumns.contains(new FieldKey(null, StoredAmount))) + selectedColumns.add(new FieldKey(null, Units)); + if (selectedColumns.contains(new FieldKey(null, ExpMaterial.ALIQUOTED_FROM_INPUT))) + selectedColumns.add(new FieldKey(null, Column.AliquotedFromLSID.name())); + if (selectedColumns.contains(new FieldKey(null, Column.IsAliquot.name()))) + selectedColumns.add(new FieldKey(null, Column.RootMaterialRowId.name())); + selectedColumns.addAll(wrappedFieldKeys); + if (null != getFilter()) + selectedColumns.addAll(getFilter().getAllFieldKeys()); + return selectedColumns; + } + + @NotNull + @Override + public SQLFragment getFromSQL(String alias) + { + return getFromSQL(alias, null); + } + + @Override + public SQLFragment getFromSQLExpanded(String alias, Set selectedColumns) + { + SQLFragment sql = new SQLFragment("("); + boolean usedMaterialized; + + + // SELECT FROM + /* NOTE We want to avoid caching in paths where the table is actively being updated (e.g. loadRows) + * Unfortunately, we don't _really_ know when this is, but if we in a transaction that's a good guess. + * Also, we may use RemapCache for material lookup outside a transaction + */ + boolean onlyMaterialColums = false; + if (null != selectedColumns && !selectedColumns.isEmpty()) + onlyMaterialColums = selectedColumns.stream().allMatch(fk -> fk.getName().equalsIgnoreCase("Folder") || null != _rootTable.getColumn(fk)); + if (!onlyMaterialColums && null != _ss && null != _ss.getTinfo() && !getExpSchema().getDbSchema().getScope().isTransactionActive()) + { + sql.append(getMaterializedSQL()); + usedMaterialized = true; + } + else + { + sql.append(getJoinSQL(selectedColumns)); + usedMaterialized = false; + } + + // WHERE + SQLFragment filterFrag = getFilter().getSQLFragment(_rootTable, null); + sql.append("\n").append(filterFrag); + if (_ss != null && !usedMaterialized) + { + if (!filterFrag.isEmpty()) + sql.append(" AND "); + else + sql.append(" WHERE "); + sql.append("CpasType = ").appendValue(_ss.getLSID()); + } + sql.append(") ").appendIdentifier(alias); + + return getTransformedFromSQL(sql); + } + + @Override + public void setSupportTableRules(boolean b) + { + this._supportTableRules = b; + } + + @Override + public boolean supportTableRules() // intentional override + { + return _supportTableRules; + } + + @Override + protected @NotNull TableRules findTableRules() + { + Container definitionContainer = getUserSchema().getContainer(); + if (null != _ss) + definitionContainer = _ss.getContainer(); + return TableRulesManager.get().getTableRules(definitionContainer, getUserSchema().getUser(), getUserSchema().getContainer()); + } + + + static class InvalidationCounters + { + public final AtomicLong update, insert, delete, rollup; + InvalidationCounters() + { + long l = System.currentTimeMillis(); + update = new AtomicLong(l); + insert = new AtomicLong(l); + delete = new AtomicLong(l); + rollup = new AtomicLong(l); + } + } + + static final BlockingCache _materializedQueries = CacheManager.getBlockingStringKeyCache(CacheManager.UNLIMITED, CacheManager.HOUR, "materialized sample types", null); + static final Map _invalidationCounters = Collections.synchronizedMap(new HashMap<>()); + static final AtomicBoolean initializedListeners = new AtomicBoolean(false); + + // used by SampleTypeServiceImpl.refreshSampleTypeMaterializedView() + public static void refreshMaterializedView(final String lsid, SampleTypeServiceImpl.SampleChangeType reason) + { + var scope = ExperimentServiceImpl.getExpSchema().getScope(); + var runnable = new RefreshMaterializedViewRunnable(lsid, reason); + scope.addCommitTask(runnable, DbScope.CommitTaskOption.POSTCOMMIT); + } + + private static class RefreshMaterializedViewRunnable implements Runnable + { + private final String _lsid; + private final SampleTypeServiceImpl.SampleChangeType _reason; + + public RefreshMaterializedViewRunnable(String lsid, SampleTypeServiceImpl.SampleChangeType reason) + { + _lsid = lsid; + _reason = reason; + } + + @Override + public void run() + { + if (_reason == schema) + { + /* NOTE: MaterializedQueryHelper can detect data changes and refresh the materialized view using the provided SQL. + * It does not handle schema changes where the SQL itself needs to be updated. In this case, we remove the + * MQH from the cache to force the SQL to be regenerated. + */ + _materializedQueries.remove(_lsid); + return; + } + var counters = getInvalidateCounters(_lsid); + switch (_reason) + { + case insert -> counters.insert.incrementAndGet(); + case rollup -> counters.rollup.incrementAndGet(); + case update -> counters.update.incrementAndGet(); + case delete -> counters.delete.incrementAndGet(); + default -> throw new IllegalStateException("Unexpected value: " + _reason); + } + } + + @Override + public boolean equals(Object obj) + { + return obj instanceof RefreshMaterializedViewRunnable other && _lsid.equals(other._lsid) && _reason.equals(other._reason); + } + } + + private static InvalidationCounters getInvalidateCounters(String lsid) + { + if (!initializedListeners.getAndSet(true)) + { + CacheManager.addListener(_invalidationCounters::clear); + } + return _invalidationCounters.computeIfAbsent(lsid, (unused) -> + new InvalidationCounters() + ); + } + + /* SELECT and JOIN, does not include WHERE, same as getJoinSQL() */ + private SQLFragment getMaterializedSQL() + { + if (null == _ss) + return getJoinSQL(null); + + var mqh = _materializedQueries.get(_ss.getLSID(), null, (unusedKey, unusedArg) -> + { + /* NOTE: MaterializedQueryHelper does have a pattern to help with detecting schema changes. + * Previously it has been used on non-provisioned tables. It might be helpful to have a pattern, + * even if just to help with race-conditions. + * + * Maybe have a callback to generate the SQL dynamically, and verify that the sql is unchanged. + */ + SQLFragment viewSql = getJoinSQL(null).append(" WHERE CpasType = ").appendValue(_ss.getLSID()); + return (_MaterializedQueryHelper) new _MaterializedQueryHelper.Builder(_ss.getLSID(), "", getExpSchema().getDbSchema().getScope(), viewSql) + .addIndex("CREATE UNIQUE INDEX uq_${NAME}_rowid ON temp.${NAME} (rowid)") + .addIndex("CREATE UNIQUE INDEX uq_${NAME}_lsid ON temp.${NAME} (lsid)") + .addIndex("CREATE INDEX idx_${NAME}_container ON temp.${NAME} (container)") + .addIndex("CREATE INDEX idx_${NAME}_root ON temp.${NAME} (rootmaterialrowid)") + .addInvalidCheck(() -> String.valueOf(getInvalidateCounters(_ss.getLSID()).update.get())) + .build(); + }); + return new SQLFragment("SELECT * FROM ").append(mqh.getFromSql("_cached_view_")); + } + + + /** + * MaterializedQueryHelper has a built-in mechanism for tracking when a temp table needs to be recomputed. + * It does not help with incremental updates (except for providing the upsert() method). + * _MaterializedQueryHelper and _Materialized copy the pattern using class Invalidator. + */ + static class _MaterializedQueryHelper extends MaterializedQueryHelper + { + final String _lsid; + + static class Builder extends MaterializedQueryHelper.Builder + { + String _lsid; + + public Builder(String lsid, String prefix, DbScope scope, SQLFragment select) + { + super(prefix, scope, select); + this._lsid = lsid; + } + + @Override + public _MaterializedQueryHelper build() + { + return new _MaterializedQueryHelper(_lsid, _prefix, _scope, _select, _uptodate, _supplier, _indexes, _max, _isSelectInto); + } + } + + _MaterializedQueryHelper(String lsid, String prefix, DbScope scope, SQLFragment select, @Nullable SQLFragment uptodate, Supplier supplier, @Nullable Collection indexes, long maxTimeToCache, + boolean isSelectIntoSql) + { + super(prefix, scope, select, uptodate, supplier, indexes, maxTimeToCache, isSelectIntoSql); + this._lsid = lsid; + } + + @Override + protected Materialized createMaterialized(String txCacheKey) + { + DbSchema temp = DbSchema.getTemp(); + String name = _prefix + "_" + GUID.makeHash(); + _Materialized materialized = new _Materialized(this, name, txCacheKey, HeartBeat.currentTimeMillis(), "\"" + temp.getName() + "\".\"" + name + "\""); + initMaterialized(materialized); + return materialized; + } + + @Override + protected void incrementalUpdateBeforeSelect(Materialized m) + { + _Materialized materialized = (_Materialized) m; + + boolean lockAcquired = false; + try + { + lockAcquired = materialized.getLock().tryLock(1, TimeUnit.MINUTES); + if (Materialized.LoadingState.ERROR == materialized._loadingState.get()) + throw materialized._loadException; + + if (!materialized.incrementalDeleteCheck.stillValid(0)) + executeIncrementalDelete(); + if (!materialized.incrementalRollupCheck.stillValid(0)) + executeIncrementalRollup(); + if (!materialized.incrementalInsertCheck.stillValid(0)) + executeIncrementalInsert(); + } + catch (RuntimeException|InterruptedException ex) + { + RuntimeException rex = UnexpectedException.wrap(ex); + materialized.setError(rex); + // The only time I'd expect an error is due to a schema change race-condition, but that can happen in any code path. + + // Ensure that next refresh starts clean + _materializedQueries.remove(_lsid); + getInvalidateCounters(_lsid).update.incrementAndGet(); + throw rex; + } + finally + { + if (lockAcquired) + materialized.getLock().unlock(); + } + } + + void upsertWithRetry(SQLFragment sql) + { + // not actually read-only, but we don't want to start an explicit transaction + _scope.executeWithRetryReadOnly((tx) -> upsert(sql)); + } + + void executeIncrementalInsert() + { + SQLFragment incremental = new SQLFragment("INSERT INTO temp.${NAME}\n") + .append("SELECT * FROM (") + .append(getViewSourceSql()).append(") viewsource_\n") + .append("WHERE rowid > (SELECT COALESCE(MAX(rowid),0) FROM temp.${NAME})"); + upsertWithRetry(incremental); + } + + void executeIncrementalDelete() + { + var d = CoreSchema.getInstance().getSchema().getSqlDialect(); + // POSTGRES bug??? the obvious query is _very_ slow O(n^2) + // DELETE FROM temp.${NAME} WHERE rowid NOT IN (SELECT rowid FROM exp.material WHERE cpastype = <<_lsid>>) + SQLFragment incremental = new SQLFragment() + .append("WITH deleted AS (SELECT rowid FROM temp.${NAME} EXCEPT SELECT rowid FROM exp.material WHERE cpastype = ").appendValue(_lsid,d).append(")\n") + .append("DELETE FROM temp.${NAME} WHERE rowid IN (SELECT rowid from deleted)\n"); + upsertWithRetry(incremental); + } + + void executeIncrementalRollup() + { + var d = CoreSchema.getInstance().getSchema().getSqlDialect(); + SQLFragment incremental = new SQLFragment(); + if (d.isPostgreSQL()) + { + incremental + .append("UPDATE temp.${NAME} AS st\n") + .append("SET aliquotcount = expm.aliquotcount, availablealiquotcount = expm.availablealiquotcount, aliquotvolume = expm.aliquotvolume, availablealiquotvolume = expm.availablealiquotvolume, aliquotunit = expm.aliquotunit\n") + .append("FROM exp.Material AS expm\n") + .append("WHERE expm.rowid = st.rowid AND expm.cpastype = ").appendValue(_lsid,d).append(" AND (\n") + .append(" st.aliquotcount IS DISTINCT FROM expm.aliquotcount OR ") + .append(" st.availablealiquotcount IS DISTINCT FROM expm.availablealiquotcount OR ") + .append(" st.aliquotvolume IS DISTINCT FROM expm.aliquotvolume OR ") + .append(" st.availablealiquotvolume IS DISTINCT FROM expm.availablealiquotvolume OR ") + .append(" st.aliquotunit IS DISTINCT FROM expm.aliquotunit") + .append(")"); + } + else + { + // SQL Server 2022 supports IS DISTINCT FROM + incremental + .append("UPDATE st\n") + .append("SET aliquotcount = expm.aliquotcount, availablealiquotcount = expm.availablealiquotcount, aliquotvolume = expm.aliquotvolume, availablealiquotvolume = expm.availablealiquotvolume, aliquotunit = expm.aliquotunit\n") + .append("FROM temp.${NAME} st, exp.Material expm\n") + .append("WHERE expm.rowid = st.rowid AND expm.cpastype = ").appendValue(_lsid,d).append(" AND (\n") + .append(" COALESCE(st.aliquotcount,-2147483648) <> COALESCE(expm.aliquotcount,-2147483648) OR ") + .append(" COALESCE(st.availablealiquotcount,-2147483648) <> COALESCE(expm.availablealiquotcount,-2147483648) OR ") + .append(" COALESCE(st.aliquotvolume,-2147483648) <> COALESCE(expm.aliquotvolume,-2147483648) OR ") + .append(" COALESCE(st.availablealiquotvolume,-2147483648) <> COALESCE(expm.availablealiquotvolume,-2147483648) OR ") + .append(" COALESCE(st.aliquotunit,'-') <> COALESCE(expm.aliquotunit,'-')") + .append(")"); + } + upsertWithRetry(incremental); + } + } + + static class _Materialized extends MaterializedQueryHelper.Materialized + { + final MaterializedQueryHelper.Invalidator incrementalInsertCheck; + final MaterializedQueryHelper.Invalidator incrementalRollupCheck; + final MaterializedQueryHelper.Invalidator incrementalDeleteCheck; + + _Materialized(_MaterializedQueryHelper mqh, String tableName, String cacheKey, long created, String sql) + { + super(mqh, tableName, cacheKey, created, sql); + final InvalidationCounters counters = getInvalidateCounters(mqh._lsid); + incrementalInsertCheck = new MaterializedQueryHelper.SupplierInvalidator(() -> String.valueOf(counters.insert.get())); + incrementalRollupCheck = new MaterializedQueryHelper.SupplierInvalidator(() -> String.valueOf(counters.rollup.get())); + incrementalDeleteCheck = new MaterializedQueryHelper.SupplierInvalidator(() -> String.valueOf(counters.delete.get())); + } + + @Override + public void reset() + { + super.reset(); + long now = HeartBeat.currentTimeMillis(); + incrementalInsertCheck.stillValid(now); + incrementalRollupCheck.stillValid(now); + incrementalDeleteCheck.stillValid(now); + } + + Lock getLock() + { + return _loadingLock; + } + } + + + /* SELECT and JOIN, does not include WHERE */ + private SQLFragment getJoinSQL(Set selectedColumns) + { + TableInfo provisioned = null == _ss ? null : _ss.getTinfo(); + Set provisionedCols = new CaseInsensitiveHashSet(provisioned != null ? provisioned.getColumnNameSet() : Collections.emptySet()); + provisionedCols.remove(Column.RowId.name()); + provisionedCols.remove(Column.LSID.name()); + provisionedCols.remove(Column.Name.name()); + boolean hasProvisionedColumns = containsProvisionedColumns(selectedColumns, provisionedCols); + + boolean hasSampleColumns = false; + boolean hasAliquotColumns = false; + + Set materialCols = new CaseInsensitiveHashSet(_rootTable.getColumnNameSet()); + selectedColumns = computeInnerSelectedColumns(selectedColumns); + + SQLFragment sql = new SQLFragment(); + sql.appendComment("", getSqlDialect()); + sql.append("SELECT "); + String comma = ""; + for (String materialCol : materialCols) + { + // don't need to generate SQL for columns that aren't selected + if (ALL_COLUMNS == selectedColumns || selectedColumns.contains(new FieldKey(null, materialCol))) + { + sql.append(comma).append("m.").appendIdentifier(materialCol); + comma = ", "; + } + } + if (null != provisioned && hasProvisionedColumns) + { + for (ColumnInfo propertyColumn : provisioned.getColumns()) + { + // don't select twice + if ( + Column.RowId.name().equalsIgnoreCase(propertyColumn.getColumnName()) || + Column.LSID.name().equalsIgnoreCase(propertyColumn.getColumnName()) || + Column.Name.name().equalsIgnoreCase(propertyColumn.getColumnName()) + ) + { + continue; + } + + // don't need to generate SQL for columns that aren't selected + if (ALL_COLUMNS == selectedColumns || selectedColumns.contains(propertyColumn.getFieldKey()) || propertyColumn.isMvIndicatorColumn()) + { + sql.append(comma); + boolean rootField = StringUtils.isEmpty(propertyColumn.getDerivationDataScope()) + || ExpSchema.DerivationDataScopeType.ParentOnly.name().equalsIgnoreCase(propertyColumn.getDerivationDataScope()); + if ("genid".equalsIgnoreCase(propertyColumn.getColumnName()) || propertyColumn.isUniqueIdField()) + { + sql.append(propertyColumn.getValueSql("m_aliquot")).append(" AS ").appendIdentifier(propertyColumn.getSelectIdentifier()); + hasAliquotColumns = true; + } + else if (rootField) + { + sql.append(propertyColumn.getValueSql("m_sample")).append(" AS ").appendIdentifier(propertyColumn.getSelectIdentifier()); + hasSampleColumns = true; + } + else + { + sql.append(propertyColumn.getValueSql("m_aliquot")).append(" AS ").appendIdentifier(propertyColumn.getSelectIdentifier()); + hasAliquotColumns = true; + } + comma = ", "; + } + } + } + + sql.append("\nFROM "); + sql.append(_rootTable, "m"); + if (hasSampleColumns) + sql.append(" INNER JOIN ").append(provisioned, "m_sample").append(" ON m.RootMaterialRowId = m_sample.RowId"); + if (hasAliquotColumns) + sql.append(" INNER JOIN ").append(provisioned, "m_aliquot").append(" ON m.RowId = m_aliquot.RowId"); + + sql.appendComment("", getSqlDialect()); + return sql; + } + + private class IdColumnRendererFactory implements DisplayColumnFactory + { + @Override + public DisplayColumn createRenderer(ColumnInfo colInfo) + { + return new IdColumnRenderer(colInfo); + } + } + + private static class IdColumnRenderer extends DataColumn + { + public IdColumnRenderer(ColumnInfo col) + { + super(col); + } + + @Override + protected boolean isDisabledInput(RenderContext ctx) + { + return !super.isDisabledInput() && ctx.getMode() != DataRegion.MODE_INSERT; + } + } + + private static class SampleTypeAmountDisplayColumn extends ExprColumn + { + public SampleTypeAmountDisplayColumn(TableInfo parent, String amountFieldName, String unitFieldName, String label, Set importAliases, Unit typeUnit) + { + super(parent, FieldKey.fromParts(amountFieldName), new SQLFragment( + "(CASE WHEN ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(unitFieldName) + .append(" = ? AND ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(amountFieldName) + .append(" IS NOT NULL THEN CAST(").append(ExprColumn.STR_TABLE_ALIAS + ".").append(amountFieldName) + .append(" / ? AS ") + .append(parent.getSqlDialect().isPostgreSQL() ? "DECIMAL" : "DOUBLE PRECISION") + .append(") ELSE ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(amountFieldName) + .append(" END)") + .add(typeUnit.getBase().toString()) + .add(typeUnit.getValue()), + JdbcType.DOUBLE); + + setLabel(label); + setImportAliasesSet(importAliases); + } + } + + private static class SampleTypeUnitDisplayColumn extends ExprColumn + { + public SampleTypeUnitDisplayColumn(TableInfo parent, String unitFieldName, Unit typeUnit) + { + super(parent, FieldKey.fromParts(Column.Units.name()), new SQLFragment( + "(CASE WHEN ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(unitFieldName) + .append(" = ? THEN ? ELSE ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(unitFieldName) + .append(" END)") + .add(typeUnit.getBase().toString()) + .add(typeUnit.toString()), + JdbcType.VARCHAR); + } + } + + @Override + public QueryUpdateService getUpdateService() + { + return new SampleTypeUpdateServiceDI(this, _ss); + } + + @Override + public boolean hasPermission(@NotNull UserPrincipal user, @NotNull Class perm) + { + if (_ss == null) + { + // Allow read and delete for exp.Materials. + // Don't allow insert/update on exp.Materials without a sample type. + if (perm == DeletePermission.class || perm == ReadPermission.class) + return getContainer().hasPermission(user, perm); + return false; + } + + if (_ss.isMedia() && perm == ReadPermission.class) + return getContainer().hasPermission(user, MediaReadPermission.class); + + return super.hasPermission(user, perm); + } + + @NotNull + @Override + public List getUniqueIndices() + { + // Rewrite the "idx_material_ak" unique index over "Folder", "SampleSet", "Name" to just "Name" + // Issue 25397: Don't include the "idx_material_ak" index if the "Name" column hasn't been added to the table. + // Some FKs to ExpMaterialTable don't include the "Name" column (e.g. NabBaseTable.Specimen) + String indexName = "idx_material_ak"; + List ret = new ArrayList<>(super.getUniqueIndices()); + if (getColumn("Name") != null) + ret.add(new IndexDefinition(indexName, IndexType.Unique, Arrays.asList(getColumn("Name")), null)); + else + ret.removeIf( def -> def.name().equals(indexName)); + return Collections.unmodifiableList(ret); + } + + + // + // UpdatableTableInfo + // + + + @Override + public @Nullable Long getOwnerObjectId() + { + return OntologyManager.ensureObject(_ss.getContainer(), _ss.getLSID(), (Long) null); + } + + @Nullable + @Override + public CaseInsensitiveHashMap remapSchemaColumns() + { + CaseInsensitiveHashMap m = new CaseInsensitiveHashMap<>(); + + if (null != getRealTable().getColumn("container") && null != getColumn("folder")) + { + m.put("container", "folder"); + } + + for (ColumnInfo col : getColumns()) + { + if (col.getMvColumnName() != null) + m.put(col.getName() + "_" + MvColumn.MV_INDICATOR_SUFFIX, col.getMvColumnName().getName()); + } + + return m; + } + + @Override + public Set getAltMergeKeys(DataIteratorContext context) + { + if (context.getInsertOption().updateOnly && context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate)) + return getAltKeysForUpdate(); + + return MATERIAL_ALT_MERGE_KEYS; + } + + @NotNull + @Override + public Set getAltKeysForUpdate() + { + return MATERIAL_ALT_UPDATE_KEYS; + } + + @Override + @NotNull + public List> getAdditionalRequiredInsertColumns() + { + if (getSampleType() == null) + return Collections.emptyList(); + + try + { + return getRequiredParentImportFields(getSampleType().getRequiredImportAliases()); + } + catch (IOException e) + { + return Collections.emptyList(); + } + } + + @Override + public DataIteratorBuilder persistRows(DataIteratorBuilder data, DataIteratorContext context) + { + TableInfo propertiesTable = _ss.getTinfo(); + + // The specimens sample type doesn't have a properties table + if (propertiesTable == null) + { + return data; + } + + long sampleTypeObjectId = requireNonNull(getOwnerObjectId()); + + // TODO: subclass PersistDataIteratorBuilder to index Materials! not DataClass! + try + { + var persist = new ExpDataIterators.PersistDataIteratorBuilder(data, this, propertiesTable, _ss, getUserSchema().getContainer(), getUserSchema().getUser(), _ss.getImportAliases(), sampleTypeObjectId) + .setFileLinkDirectory(SAMPLETYPE_FILE_DIRECTORY); + ExperimentServiceImpl experimentServiceImpl = ExperimentServiceImpl.get(); + SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(getContainer(), SearchService.PRIORITY.modified); + + persist.setIndexFunction(searchIndexDataKeys -> propertiesTable.getSchema().getScope().addCommitTask(() -> + { + List lsids = searchIndexDataKeys.lsids(); + List orderedRowIds = searchIndexDataKeys.orderedRowIds(); + + // Issue 51263: order by RowId to reduce deadlock + ListUtils.partition(orderedRowIds, 100).forEach(sublist -> + queue.addRunnable((q) -> + { + for (ExpMaterialImpl expMaterial : experimentServiceImpl.getExpMaterials(sublist)) + expMaterial.index(q, this); + }) + ); + + ListUtils.partition(lsids, 100).forEach(sublist -> + queue.addRunnable((q) -> + { + for (ExpMaterialImpl expMaterial : experimentServiceImpl.getExpMaterialsByLsid(sublist)) + expMaterial.index(q, this); + }) + ); + }, DbScope.CommitTaskOption.POSTCOMMIT) + ); + + DataIteratorBuilder builder = LoggingDataIterator.wrap(persist); + return LoggingDataIterator.wrap(new AliasDataIteratorBuilder(builder, getUserSchema().getContainer(), getUserSchema().getUser(), ExperimentService.get().getTinfoMaterialAliasMap(), _ss, true)); + } + catch (IOException e) + { + throw new UncheckedIOException(e); + } + } + + @Override + @NotNull + public AuditBehaviorType getDefaultAuditBehavior() + { + return AuditBehaviorType.DETAILED; + } + + static final Set excludeFromDetailedAuditField; + static + { + var set = new CaseInsensitiveHashSet(); + set.addAll(TableInfo.defaultExcludedDetailedUpdateAuditFields); + set.addAll(ExpDataIterators.NOT_FOR_UPDATE); + // We don't want the inventory columns to show up in the sample timeline audit record; + // they are captured in their own audit record. + set.addAll(InventoryService.InventoryStatusColumn.names()); + excludeFromDetailedAuditField = Collections.unmodifiableSet(set); + } + + @Override + public @NotNull Set getExcludedDetailedUpdateAuditFields() + { + // uniqueId fields don't change in reality, so exclude them from the audit updates + Set excluded = new CaseInsensitiveHashSet(); + excluded.addAll(this.getUniqueIdFields()); + excluded.addAll(excludeFromDetailedAuditField); + return excluded; + } + + @Override + public List> getImportTemplates(ViewContext ctx) + { + // respect any metadata overrides + if (getRawImportTemplates() != null) + return super.getImportTemplates(ctx); + + List> templates = new ArrayList<>(); + ActionURL url = PageFlowUtil.urlProvider(QueryUrls.class).urlCreateExcelTemplate(ctx.getContainer(), getPublicSchemaName(), getName()); + url.addParameter("headerType", ColumnHeaderType.ImportField.name()); + try + { + if (getSampleType() != null && !getSampleType().getImportAliases().isEmpty()) + { + for (String aliasKey : getSampleType().getImportAliases().keySet()) + url.addParameter("includeColumn", aliasKey); + } + } + catch (IOException e) + {} + templates.add(Pair.of("Download Template", url.toString())); + return templates; + } + + @Override + public void overlayMetadata(String tableName, UserSchema schema, Collection errors) + { + if (SamplesSchema.SCHEMA_NAME.equals(schema.getName())) + { + Collection metadata = QueryService.get().findMetadataOverride(schema, SamplesSchema.SCHEMA_METADATA_NAME, false, false, errors, null); + if (null != metadata) + { + overlayMetadata(metadata, schema, errors); + } + } + super.overlayMetadata(tableName, schema, errors); + } + + static class SampleTypeAmountPrecisionDisplayColumn extends DataColumn + { + Unit typeUnit; + boolean applySampleTypePrecision = true; + + public SampleTypeAmountPrecisionDisplayColumn(ColumnInfo col, Unit typeUnit) { + super(col, false); + this.typeUnit = typeUnit; + this.applySampleTypePrecision = col.getFormat() == null; // only apply if no custom format is set by user + } + + @Override + public Object getDisplayValue(RenderContext ctx) + { + Object value = super.getDisplayValue(ctx); + if (this.applySampleTypePrecision && value != null) + { + int scale = this.typeUnit == null ? Quantity.DEFAULT_PRECISION_SCALE : this.typeUnit.getPrecisionScale(); + value = Precision.round(Double.valueOf(value.toString()), scale); + } + return value; + } + } +} From a471fb185832bdf9a361ffdb73aa90af89a56a47 Mon Sep 17 00:00:00 2001 From: cnathe Date: Mon, 6 Oct 2025 09:51:33 -0500 Subject: [PATCH 2/3] attempt restore line ending --- .../experiment/api/ExpMaterialTableImpl.java | 3712 ++++++++--------- 1 file changed, 1856 insertions(+), 1856 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index 94d8fd0bfb0..15c26069d6e 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -1,1856 +1,1856 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.experiment.api; - -import org.apache.commons.collections4.ListUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.math3.util.Precision; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.assay.plate.AssayPlateMetadataService; -import org.labkey.api.audit.AuditHandler; -import org.labkey.api.cache.BlockingCache; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.compliance.TableRules; -import org.labkey.api.compliance.TableRulesManager; -import org.labkey.api.data.ColumnHeaderType; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DataColumn; -import org.labkey.api.data.DataRegion; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.DisplayColumn; -import org.labkey.api.data.DisplayColumnFactory; -import org.labkey.api.data.ForeignKey; -import org.labkey.api.data.ImportAliasable; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.MaterializedQueryHelper; -import org.labkey.api.data.MutableColumnInfo; -import org.labkey.api.data.PHI; -import org.labkey.api.data.RenderContext; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.Sort; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.UnionContainerFilter; -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.dataiterator.DataIteratorContext; -import org.labkey.api.dataiterator.LoggingDataIterator; -import org.labkey.api.dataiterator.SimpleTranslator; -import org.labkey.api.exp.MvColumn; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.PropertyColumn; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.ExperimentUrls; -import org.labkey.api.exp.api.NameExpressionOptionService; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.DomainUtil; -import org.labkey.api.exp.query.ExpDataTable; -import org.labkey.api.exp.query.ExpMaterialTable; -import org.labkey.api.exp.query.ExpSampleTypeTable; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.exp.query.SamplesSchema; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.inventory.InventoryService; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.ontology.Unit; -import org.labkey.api.qc.SampleStatusService; -import org.labkey.api.query.AliasedColumn; -import org.labkey.api.query.DetailsURL; -import org.labkey.api.query.ExprColumn; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.LookupForeignKey; -import org.labkey.api.query.QueryException; -import org.labkey.api.query.QueryForeignKey; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QueryUpdateService; -import org.labkey.api.query.QueryUrls; -import org.labkey.api.query.RowIdForeignKey; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.UserSchema; -import org.labkey.api.query.column.BuiltInColumnTypes; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.UserPrincipal; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.InsertPermission; -import org.labkey.api.security.permissions.MediaReadPermission; -import org.labkey.api.security.permissions.MoveEntitiesPermission; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.util.GUID; -import org.labkey.api.util.HeartBeat; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.StringExpression; -import org.labkey.api.util.UnexpectedException; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.ViewContext; -import org.labkey.data.xml.TableType; -import org.labkey.experiment.ExpDataIterators; -import org.labkey.experiment.ExpDataIterators.AliasDataIteratorBuilder; -import org.labkey.experiment.controllers.exp.ExperimentController; -import org.labkey.experiment.lineage.LineageMethod; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static java.util.Objects.requireNonNull; -import static org.labkey.api.exp.api.SampleTypeDomainKind.ALIQUOT_COUNT_LABEL; -import static org.labkey.api.exp.api.SampleTypeDomainKind.ALIQUOT_VOLUME_LABEL; -import static org.labkey.api.exp.api.SampleTypeDomainKind.AVAILABLE_ALIQUOT_COUNT_LABEL; -import static org.labkey.api.exp.api.SampleTypeDomainKind.AVAILABLE_ALIQUOT_VOLUME_LABEL; -import static org.labkey.api.exp.api.SampleTypeDomainKind.SAMPLETYPE_FILE_DIRECTORY; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotCount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotVolume; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AvailableAliquotCount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AvailableAliquotVolume; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; -import static org.labkey.api.util.StringExpressionFactory.AbstractStringExpression.NullValueBehavior.NullResult; -import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.schema; - -public class ExpMaterialTableImpl extends ExpRunItemTableImpl implements ExpMaterialTable -{ - ExpSampleTypeImpl _ss; - Set _uniqueIdFields; - boolean _supportTableRules = true; - - public static final Set MATERIAL_ALT_MERGE_KEYS; - public static final Set MATERIAL_ALT_UPDATE_KEYS; - static { - MATERIAL_ALT_MERGE_KEYS = Set.of(Column.MaterialSourceId.name(), Column.Name.name()); - MATERIAL_ALT_UPDATE_KEYS = Set.of(Column.LSID.name()); - } - - public ExpMaterialTableImpl(UserSchema schema, ContainerFilter cf, @Nullable ExpSampleType sampleType) - { - super(ExpSchema.TableType.Materials.name(), ExperimentServiceImpl.get().getTinfoMaterial(), schema, cf); - setDetailsURL(new DetailsURL(new ActionURL(ExperimentController.ShowMaterialAction.class, schema.getContainer()), Collections.singletonMap("rowId", "rowId"), NullResult)); - setPublicSchemaName(ExpSchema.SCHEMA_NAME); - addAllowablePermission(InsertPermission.class); - addAllowablePermission(UpdatePermission.class); - addAllowablePermission(MoveEntitiesPermission.class); - setAllowedInsertOption(QueryUpdateService.InsertOption.MERGE); - setSampleType(sampleType); - } - - public Set getUniqueIdFields() - { - if (_uniqueIdFields == null) - { - _uniqueIdFields = new CaseInsensitiveHashSet(); - _uniqueIdFields.addAll(getColumns().stream().filter(ColumnInfo::isUniqueIdField).map(ColumnInfo::getName).collect(Collectors.toSet())); - } - return _uniqueIdFields; - } - - @Override - protected ColumnInfo resolveColumn(String name) - { - ColumnInfo result = super.resolveColumn(name); - if (result == null) - { - if ("CpasType".equalsIgnoreCase(name)) - result = createColumn(Column.SampleSet.name(), Column.SampleSet); - else if (Column.Property.name().equalsIgnoreCase(name)) - result = createPropertyColumn(Column.Property.name()); - else if (Column.QueryableInputs.name().equalsIgnoreCase(name)) - result = createColumn(Column.QueryableInputs.name(), Column.QueryableInputs); - } - return result; - } - - @Override - public ColumnInfo getExpObjectColumn() - { - var ret = wrapColumn("ExpMaterialTableImpl_object_", _rootTable.getColumn("objectid")); - ret.setConceptURI(BuiltInColumnTypes.EXPOBJECTID_CONCEPT_URI); - return ret; - } - - @Override - public AuditHandler getAuditHandler(AuditBehaviorType auditBehaviorType) - { - if (getUserSchema().getName().equalsIgnoreCase(SamplesSchema.SCHEMA_NAME)) - { - // Special case sample auditing to help build a useful timeline view - return SampleTypeServiceImpl.get(); - } - - return super.getAuditHandler(auditBehaviorType); - } - - @Override - public MutableColumnInfo createColumn(String alias, Column column) - { - switch (column) - { - case Folder -> - { - return wrapColumn(alias, _rootTable.getColumn("Container")); - } - case LSID -> - { - return wrapColumn(alias, _rootTable.getColumn(Column.LSID.name())); - } - case MaterialSourceId -> - { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.MaterialSourceId.name())); - columnInfo.setFk(new LookupForeignKey(getLookupContainerFilter(), null, null, null, null, "RowId", "Name") - { - @Override - public TableInfo getLookupTableInfo() - { - ExpSampleTypeTable sampleTypeTable = ExperimentService.get().createSampleTypeTable(ExpSchema.TableType.SampleSets.toString(), _userSchema, getLookupContainerFilter()); - sampleTypeTable.populate(); - return sampleTypeTable; - } - - @Override - public StringExpression getURL(ColumnInfo parent) - { - return super.getURL(parent, true); - } - }); - columnInfo.setUserEditable(false); - columnInfo.setReadOnly(true); - columnInfo.setHidden(true); - return columnInfo; - } - case RootMaterialRowId -> - { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.RootMaterialRowId.name())); - columnInfo.setFk(getExpSchema().getMaterialForeignKey(getLookupContainerFilter(), Column.RowId.name())); - columnInfo.setLabel("Root Material"); - columnInfo.setUserEditable(false); - - // NK: Here we mark the column as not required AND nullable which is the opposite of the database where - // a NOT NULL constraint is in place. This is done to avoid the RequiredValidator check upon updating a row. - // See ExpMaterialValidatorIterator. - columnInfo.setRequired(false); - columnInfo.setNullable(true); - - return columnInfo; - } - case AliquotedFromLSID -> - { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.AliquotedFromLSID.name())); - columnInfo.setSqlTypeName("lsidtype"); - columnInfo.setFk(getExpSchema().getMaterialForeignKey(getLookupContainerFilter(), Column.LSID.name())); - columnInfo.setLabel("Aliquoted From Parent"); - return columnInfo; - } - case IsAliquot -> - { - String rootMaterialRowIdField = ExprColumn.STR_TABLE_ALIAS + "." + Column.RootMaterialRowId.name(); - String rowIdField = ExprColumn.STR_TABLE_ALIAS + "." + Column.RowId.name(); - ExprColumn columnInfo = new ExprColumn(this, FieldKey.fromParts(Column.IsAliquot.name()), new SQLFragment( - "(CASE WHEN (" + rootMaterialRowIdField + " = " + rowIdField + ") THEN ").append(getSqlDialect().getBooleanFALSE()) - .append(" WHEN ").append(rowIdField).append(" IS NOT NULL THEN ").append(getSqlDialect().getBooleanTRUE()) // Issue 52745 - .append(" ELSE NULL END)"), JdbcType.BOOLEAN); - columnInfo.setLabel("Is Aliquot"); - columnInfo.setDescription("Identifies if the material is a sample or an aliquot"); - columnInfo.setUserEditable(false); - columnInfo.setReadOnly(true); - columnInfo.setHidden(false); - return columnInfo; - } - case Name -> - { - var nameCol = wrapColumn(alias, _rootTable.getColumn(column.toString())); - // shut off this field in insert and update views if user specified names are not allowed - if (!NameExpressionOptionService.get().getAllowUserSpecificNamesValue(getContainer())) - { - nameCol.setShownInInsertView(false); - nameCol.setShownInUpdateView(false); - } - return nameCol; - } - case RawAmount -> - { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.StoredAmount.name())); - columnInfo.setDisplayColumnFactory(colInfo -> new SampleTypeAmountPrecisionDisplayColumn(colInfo, null)); - columnInfo.setDescription("The amount of this sample, in the base unit for the sample type's display unit (if defined), currently on hand."); - columnInfo.setUserEditable(false); - columnInfo.setReadOnly(true); - return columnInfo; - } - case StoredAmount -> - { - String label = StoredAmount.label(); - Set importAliases = Set.of(label, "Stored Amount"); - Unit typeUnit = getSampleTypeUnit(); - if (typeUnit != null) - { - SampleTypeAmountDisplayColumn columnInfo = new SampleTypeAmountDisplayColumn(this, Column.StoredAmount.name(), Column.Units.name(), label, importAliases, typeUnit); - columnInfo.setDisplayColumnFactory(colInfo -> new SampleTypeAmountPrecisionDisplayColumn(colInfo, typeUnit)); - columnInfo.setDescription("The amount of this sample, in the display unit for the sample type, currently on hand."); - columnInfo.setShownInUpdateView(true); - columnInfo.setShownInInsertView(true); - columnInfo.setUserEditable(true); - columnInfo.setCalculated(false); - return columnInfo; - } - else - { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.StoredAmount.name())); - columnInfo.setDisplayColumnFactory(colInfo -> new SampleTypeAmountPrecisionDisplayColumn(colInfo, null)); - columnInfo.setLabel(label); - columnInfo.setImportAliasesSet(importAliases); - columnInfo.setDescription("The amount of this sample currently on hand."); - return columnInfo; - } - } - case RawUnits -> - { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.Units.name())); - columnInfo.setDescription("The units associated with the Stored Amount for this sample."); - columnInfo.setUserEditable(false); - columnInfo.setReadOnly(true); - return columnInfo; - } - case Units -> - { - ForeignKey fk = new LookupForeignKey("Value", "Value") - { - @Override - public @Nullable TableInfo getLookupTableInfo() - { - return getExpSchema().getTable(ExpSchema.MEASUREMENT_UNITS_TABLE); - } - }; - - Unit typeUnit = getSampleTypeUnit(); - if (typeUnit != null) - { - SampleTypeUnitDisplayColumn columnInfo = new SampleTypeUnitDisplayColumn(this, Column.Units.name(), typeUnit); - columnInfo.setFk(fk); - columnInfo.setDescription("The sample type display units associated with the Amount for this sample."); - columnInfo.setShownInUpdateView(true); - columnInfo.setShownInInsertView(true); - columnInfo.setUserEditable(true); - columnInfo.setCalculated(false); - return columnInfo; - } - else - { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.Units.name())); - columnInfo.setFk(fk); - columnInfo.setDescription("The units associated with the Stored Amount for this sample."); - return columnInfo; - } - } - case Description -> - { - return wrapColumn(alias, _rootTable.getColumn(Column.Description.name())); - } - case SampleSet -> - { - var columnInfo = wrapColumn(alias, _rootTable.getColumn("CpasType")); - // NOTE: populateColumns() overwrites this with a QueryForeignKey. Can this be removed? - columnInfo.setFk(new LookupForeignKey(getContainerFilter(), null, null, null, null, "LSID", "Name") - { - @Override - public TableInfo getLookupTableInfo() - { - ExpSampleTypeTable sampleTypeTable = ExperimentService.get().createSampleTypeTable(ExpSchema.TableType.SampleSets.toString(), _userSchema, getLookupContainerFilter()); - sampleTypeTable.populate(); - return sampleTypeTable; - } - - @Override - public StringExpression getURL(ColumnInfo parent) - { - return super.getURL(parent, true); - } - }); - return columnInfo; - } - case SourceProtocolLSID -> - { - // NOTE: This column is incorrectly named "Protocol", but we are keeping it for backwards compatibility to avoid breaking queries in hvtnFlow module - ExprColumn columnInfo = new ExprColumn(this, ExpDataTable.Column.Protocol.toString(), new SQLFragment( - "(SELECT ProtocolLSID FROM " + ExperimentServiceImpl.get().getTinfoProtocolApplication() + " pa " + - " WHERE pa.RowId = " + ExprColumn.STR_TABLE_ALIAS + ".SourceApplicationId)"), JdbcType.VARCHAR); - columnInfo.setSqlTypeName("lsidtype"); - columnInfo.setFk(getExpSchema().getProtocolForeignKey(getContainerFilter(), "LSID")); - columnInfo.setLabel("Source Protocol"); - columnInfo.setDescription("Contains a reference to the protocol for the protocol application that created this sample"); - columnInfo.setUserEditable(false); - columnInfo.setReadOnly(true); - columnInfo.setHidden(true); - return columnInfo; - } - case SourceProtocolApplication -> - { - var columnInfo = wrapColumn(alias, _rootTable.getColumn("SourceApplicationId")); - columnInfo.setFk(getExpSchema().getProtocolApplicationForeignKey(getContainerFilter())); - columnInfo.setUserEditable(false); - columnInfo.setReadOnly(true); - columnInfo.setHidden(true); - columnInfo.setAutoIncrement(false); - return columnInfo; - } - case SourceApplicationInput -> - { - var col = createEdgeColumn(alias, Column.SourceProtocolApplication, ExpSchema.TableType.MaterialInputs); - col.setDescription("Contains a reference to the MaterialInput row between this ExpMaterial and it's SourceProtocolApplication"); - col.setHidden(true); - return col; - } - case RunApplication -> - { - SQLFragment sql = new SQLFragment("(SELECT pa.rowId FROM ") - .append(ExperimentService.get().getTinfoProtocolApplication(), "pa") - .append(" WHERE pa.runId = ").append(ExprColumn.STR_TABLE_ALIAS).append(".runId") - .append(" AND pa.cpasType = ").appendValue(ExpProtocol.ApplicationType.ExperimentRunOutput) - .append(")"); - - var col = new ExprColumn(this, alias, sql, JdbcType.INTEGER); - col.setFk(getExpSchema().getProtocolApplicationForeignKey(getContainerFilter())); - col.setDescription("Contains a reference to the ExperimentRunOutput protocol application of the run that created this sample"); - col.setUserEditable(false); - col.setReadOnly(true); - col.setHidden(true); - return col; - } - case RunApplicationOutput -> - { - var col = createEdgeColumn(alias, Column.RunApplication, ExpSchema.TableType.MaterialInputs); - col.setDescription("Contains a reference to the MaterialInput row between this ExpMaterial and it's RunOutputApplication"); - return col; - } - case Run -> - { - var ret = wrapColumn(alias, _rootTable.getColumn("RunId")); - ret.setReadOnly(true); - return ret; - } - case RowId -> - { - var ret = wrapColumn(alias, _rootTable.getColumn("RowId")); - // When no sorts are added by views, QueryServiceImpl.createDefaultSort() adds the primary key's default sort direction - ret.setSortDirection(Sort.SortDirection.DESC); - ret.setFk(new RowIdForeignKey(ret)); - ret.setUserEditable(false); - ret.setHidden(true); - ret.setShownInInsertView(false); - ret.setHasDbSequence(true); - ret.setIsRootDbSequence(true); - return ret; - } - case Property -> - { - return createPropertyColumn(alias); - } - case Flag -> - { - return createFlagColumn(alias); - } - case Created -> - { - return wrapColumn(alias, _rootTable.getColumn("Created")); - } - case CreatedBy -> - { - return createUserColumn(alias, _rootTable.getColumn("CreatedBy")); - } - case Modified -> - { - return wrapColumn(alias, _rootTable.getColumn("Modified")); - } - case ModifiedBy -> - { - return createUserColumn(alias, _rootTable.getColumn("ModifiedBy")); - } - case Alias -> - { - return createAliasColumn(alias, ExperimentService.get()::getTinfoMaterialAliasMap); - } - case Inputs -> - { - return createLineageColumn(this, alias, true, false); - } - case QueryableInputs -> - { - return createLineageColumn(this, alias, true, true); - } - case Outputs -> - { - return createLineageColumn(this, alias, false, false); - } - case Properties -> - { - return createPropertiesColumn(alias); - } - case SampleState -> - { - boolean statusEnabled = SampleStatusService.get().supportsSampleStatus() && !SampleStatusService.get().getAllProjectStates(getContainer()).isEmpty(); - var ret = wrapColumn(alias, _rootTable.getColumn(column.name())); - ret.setLabel("Status"); - ret.setHidden(!statusEnabled); - ret.setShownInDetailsView(statusEnabled); - ret.setShownInInsertView(statusEnabled); - ret.setShownInUpdateView(statusEnabled); - ret.setRemapMissingBehavior(SimpleTranslator.RemapMissingBehavior.Error); - ret.setFk(new QueryForeignKey.Builder(getUserSchema(), getSampleStatusLookupContainerFilter()) - .schema(getExpSchema()).table(ExpSchema.TableType.SampleStatus).display("Label")); - return ret; - } - case AliquotCount -> - { - var ret = wrapColumn(alias, _rootTable.getColumn(AliquotCount.name())); - ret.setLabel(ALIQUOT_COUNT_LABEL); - return ret; - } - case AliquotVolume -> - { - var ret = wrapColumn(alias, _rootTable.getColumn(AliquotVolume.name())); - ret.setLabel(ALIQUOT_VOLUME_LABEL); - return ret; - } - case AvailableAliquotVolume -> - { - var ret = wrapColumn(alias, _rootTable.getColumn(AvailableAliquotVolume.name())); - ret.setLabel(AVAILABLE_ALIQUOT_VOLUME_LABEL); - return ret; - } - case AvailableAliquotCount -> - { - var ret = wrapColumn(alias, _rootTable.getColumn(AvailableAliquotCount.name())); - ret.setLabel(AVAILABLE_ALIQUOT_COUNT_LABEL); - return ret; - } - case AliquotUnit -> - { - var ret = wrapColumn(alias, _rootTable.getColumn("AliquotUnit")); - ret.setShownInDetailsView(false); - return ret; - } - case MaterialExpDate -> - { - var ret = wrapColumn(alias, _rootTable.getColumn("MaterialExpDate")); - ret.setLabel("Expiration Date"); - ret.setShownInDetailsView(true); - ret.setShownInInsertView(true); - ret.setShownInUpdateView(true); - return ret; - } - default -> throw new IllegalArgumentException("Unknown column " + column); - } - } - - @Override - public MutableColumnInfo createPropertyColumn(String alias) - { - var ret = super.createPropertyColumn(alias); - if (_ss != null) - { - final TableInfo t = _ss.getTinfo(); - if (t != null) - { - ret.setFk(new LookupForeignKey() - { - @Override - public TableInfo getLookupTableInfo() - { - return t; - } - - @Override - protected ColumnInfo getPkColumn(TableInfo table) - { - return t.getColumn("lsid"); - } - }); - } - } - ret.setIsUnselectable(true); - ret.setDescription("A holder for any custom fields associated with this sample"); - ret.setHidden(true); - return ret; - } - - private Unit getSampleTypeUnit() - { - Unit typeUnit = null; - if (_ss != null && _ss.getMetricUnit() != null) - typeUnit = Unit.fromName(_ss.getMetricUnit()); - return typeUnit; - } - - private void setSampleType(@Nullable ExpSampleType st) - { - checkLocked(); - if (_ss != null) - { - throw new IllegalStateException("Cannot unset sample type"); - } - if (st != null && !(st instanceof ExpSampleTypeImpl)) - { - throw new IllegalArgumentException("Expected sample type to be an instance of " + ExpSampleTypeImpl.class.getName() + " but was a " + st.getClass().getName()); - } - _ss = (ExpSampleTypeImpl) st; - if (_ss != null) - { - setPublicSchemaName(SamplesSchema.SCHEMA_NAME); - setName(st.getName()); - - String description = _ss.getDescription(); - if (StringUtils.isEmpty(description)) - description = "Contains one row per sample in the " + _ss.getName() + " sample type"; - setDescription(description); - - if (canUserAccessPhi()) - { - ActionURL url = PageFlowUtil.urlProvider(ExperimentUrls.class).getImportSamplesURL(getContainer(), _ss.getName()); - setImportURL(new DetailsURL(url)); - } - } - } - - public ExpSampleType getSampleType() - { - return _ss; - } - - @Override - protected void populateColumns() - { - var st = getSampleType(); - var rowIdCol = addColumn(Column.RowId); - addColumn(Column.MaterialSourceId); - addColumn(Column.SourceProtocolApplication); - addColumn(Column.SourceApplicationInput); - addColumn(Column.RunApplication); - addColumn(Column.RunApplicationOutput); - addColumn(Column.SourceProtocolLSID); - - var nameCol = addColumn(Column.Name); - if (st != null && st.hasNameAsIdCol()) - { - // Show the Name field but don't mark is as required when using name expressions - if (st.hasNameExpression()) - { - var nameExpression = st.getNameExpression(); - nameCol.setNameExpression(nameExpression); - nameCol.setNullable(true); - String nameExpressionPreview = getExpNameExpressionPreview(getUserSchema().getSchemaName(), st.getName(), getUserSchema().getUser()); - String desc = appendNameExpressionDescription(nameCol.getDescription(), nameExpression, nameExpressionPreview); - nameCol.setDescription(desc); - } - else - { - nameCol.setNullable(false); - } - } - else - { - nameCol.setReadOnly(true); - nameCol.setShownInInsertView(false); - } - - addColumn(Column.Alias); - addColumn(Column.Description); - - var typeColumnInfo = addColumn(Column.SampleSet); - typeColumnInfo.setFk(new QueryForeignKey(_userSchema, getContainerFilter(), ExpSchema.SCHEMA_NAME, getContainer(), null, ExpSchema.TableType.SampleSets.name(), "lsid", null) - { - @Override - protected ContainerFilter getLookupContainerFilter() - { - // Be sure that we can resolve the sample type if it's defined in a separate container. - // Same as CurrentPlusProjectAndShared but includes SampleSet's container as well. - // Issue 37982: Sample Type: Link to precursor sample type does not resolve correctly if sample has - // parents in current sample type and a sample type in the parent container - Set containers = new HashSet<>(); - if (null != st) - containers.add(st.getContainer()); - containers.add(getContainer()); - if (getContainer().getProject() != null) - containers.add(getContainer().getProject()); - containers.add(ContainerManager.getSharedContainer()); - ContainerFilter cf = new ContainerFilter.CurrentPlusExtras(_userSchema.getContainer(), _userSchema.getUser(), containers); - - if (null != _containerFilter && _containerFilter.getType() != ContainerFilter.Type.Current) - cf = new UnionContainerFilter(_containerFilter, cf); - return cf; - } - }); - - typeColumnInfo.setReadOnly(true); - typeColumnInfo.setUserEditable(false); - typeColumnInfo.setShownInInsertView(false); - - addColumn(Column.MaterialExpDate); - addContainerColumn(Column.Folder, null); - var runCol = addColumn(Column.Run); - runCol.setFk(new ExpSchema(_userSchema.getUser(), getContainer()).getRunIdForeignKey(getContainerFilter())); - runCol.setShownInInsertView(false); - runCol.setShownInUpdateView(false); - - var colLSID = addColumn(Column.LSID); - colLSID.setHidden(true); - colLSID.setReadOnly(true); - colLSID.setUserEditable(false); - colLSID.setShownInInsertView(false); - colLSID.setShownInDetailsView(false); - colLSID.setShownInUpdateView(false); - - var rootRowId = addColumn(Column.RootMaterialRowId); - rootRowId.setHidden(true); - rootRowId.setReadOnly(true); - rootRowId.setUserEditable(false); - rootRowId.setShownInInsertView(false); - rootRowId.setShownInDetailsView(false); - rootRowId.setShownInUpdateView(false); - - var aliquotParentLSID = addColumn(Column.AliquotedFromLSID); - aliquotParentLSID.setHidden(true); - aliquotParentLSID.setReadOnly(true); - aliquotParentLSID.setUserEditable(false); - aliquotParentLSID.setShownInInsertView(false); - aliquotParentLSID.setShownInDetailsView(false); - aliquotParentLSID.setShownInUpdateView(false); - - addColumn(Column.IsAliquot); - addColumn(Column.Created); - addColumn(Column.CreatedBy); - addColumn(Column.Modified); - addColumn(Column.ModifiedBy); - - List defaultCols = new ArrayList<>(); - defaultCols.add(FieldKey.fromParts(Column.Name)); - defaultCols.add(FieldKey.fromParts(Column.MaterialExpDate)); - boolean hasProductFolders = getContainer().hasProductFolders(); - if (hasProductFolders) - defaultCols.add(FieldKey.fromParts(Column.Folder)); - defaultCols.add(FieldKey.fromParts(Column.Run)); - - if (st == null) - defaultCols.add(FieldKey.fromParts(Column.SampleSet)); - - addColumn(Column.Flag); - - var statusColInfo = addColumn(Column.SampleState); - boolean statusEnabled = SampleStatusService.get().supportsSampleStatus() && !SampleStatusService.get().getAllProjectStates(getContainer()).isEmpty(); - statusColInfo.setShownInDetailsView(statusEnabled); - statusColInfo.setShownInInsertView(statusEnabled); - statusColInfo.setShownInUpdateView(statusEnabled); - statusColInfo.setHidden(!statusEnabled); - statusColInfo.setRemapMissingBehavior(SimpleTranslator.RemapMissingBehavior.Error); - if (statusEnabled) - defaultCols.add(FieldKey.fromParts(Column.SampleState)); - statusColInfo.setFk(new QueryForeignKey.Builder(getUserSchema(), getSampleStatusLookupContainerFilter()) - .schema(getExpSchema()).table(ExpSchema.TableType.SampleStatus).display("Label")); - - // TODO is this a real Domain??? - if (st != null && !"urn:lsid:labkey.com:SampleSource:Default".equals(st.getDomain().getTypeURI())) - { - defaultCols.add(FieldKey.fromParts(Column.Flag)); - addSampleTypeColumns(st, defaultCols); - - setName(_ss.getName()); - - ActionURL gridUrl = new ActionURL(ExperimentController.ShowSampleTypeAction.class, getContainer()); - gridUrl.addParameter("rowId", st.getRowId()); - setGridURL(new DetailsURL(gridUrl)); - } - - List calculatedFieldKeys = DomainUtil.getCalculatedFieldsForDefaultView(this); - defaultCols.addAll(calculatedFieldKeys); - - addColumn(Column.AliquotCount); - addColumn(Column.AliquotVolume); - addColumn(Column.AliquotUnit); - addColumn(Column.AvailableAliquotCount); - addColumn(Column.AvailableAliquotVolume); - - addColumn(Column.StoredAmount); - defaultCols.add(FieldKey.fromParts(Column.StoredAmount)); - - addColumn(Column.Units); - defaultCols.add(FieldKey.fromParts(Column.Units)); - - var rawAmountColumn = addColumn(Column.RawAmount); - rawAmountColumn.setDisplayColumnFactory(new DisplayColumnFactory() - { - @Override - public DisplayColumn createRenderer(ColumnInfo colInfo) - { - return new DataColumn(colInfo) - { - @Override - public void addQueryFieldKeys(Set keys) - { - super.addQueryFieldKeys(keys); - keys.add(FieldKey.fromParts(Column.StoredAmount)); - - } - }; - } - }); - rawAmountColumn.setHidden(true); - rawAmountColumn.setShownInDetailsView(false); - rawAmountColumn.setShownInInsertView(false); - rawAmountColumn.setShownInUpdateView(false); - - var rawUnitsColumn = addColumn(Column.RawUnits); - rawUnitsColumn.setDisplayColumnFactory(new DisplayColumnFactory() - { - @Override - public DisplayColumn createRenderer(ColumnInfo colInfo) - { - return new DataColumn(colInfo) - { - @Override - public void addQueryFieldKeys(Set keys) - { - super.addQueryFieldKeys(keys); - keys.add(FieldKey.fromParts(Column.Units)); - - } - }; - } - }); - rawUnitsColumn.setHidden(true); - rawUnitsColumn.setShownInDetailsView(false); - rawUnitsColumn.setShownInInsertView(false); - rawUnitsColumn.setShownInUpdateView(false); - - if (InventoryService.get() != null && (st == null || !st.isMedia())) - defaultCols.addAll(InventoryService.get().addInventoryStatusColumns(st == null ? null : st.getMetricUnit(), this, getContainer(), _userSchema.getUser())); - - SQLFragment sql; - UserSchema plateUserSchema; - // Issue 53194 : this would be the case for linked to study samples. The contextual role is set up from the study dataset - // for the source sample, we want to allow the plate schema to inherit any contextual roles to allow querying - // against tables in that schema. - if (_userSchema instanceof UserSchema.HasContextualRoles samplesSchema && !samplesSchema.getContextualRoles().isEmpty()) - plateUserSchema = AssayPlateMetadataService.get().getPlateSchema(_userSchema, samplesSchema.getContextualRoles()); - else - plateUserSchema = QueryService.get().getUserSchema(_userSchema.getUser(), _userSchema.getContainer(), "plate"); - - if (plateUserSchema != null && plateUserSchema.getTable("Well") != null) - { - String rowIdField = ExprColumn.STR_TABLE_ALIAS + "." + Column.RowId.name(); - SQLFragment existsSubquery = new SQLFragment() - .append("SELECT 1 FROM ") - .append(plateUserSchema.getTable("Well"), "well") - .append(" WHERE well.sampleid = ").append(rowIdField); - - sql = new SQLFragment() - .append("CASE WHEN EXISTS (") - .append(existsSubquery) - .append(") THEN 'Plated' ") - .append("WHEN ").append(ExprColumn.STR_TABLE_ALIAS).append(".RowId").append(" IS NOT NULL THEN 'Not Plated' ")// Issue 52745 - .append("ELSE NULL END"); - } - else - { - sql = new SQLFragment("(SELECT NULL)"); - } - var col = new ExprColumn(this, Column.IsPlated.name(), sql, JdbcType.VARCHAR); - col.setDescription("Whether the sample that has been plated, if plating is supported."); - col.setUserEditable(false); - col.setReadOnly(true); - col.setShownInDetailsView(false); - col.setShownInInsertView(false); - col.setShownInUpdateView(false); - if (plateUserSchema != null) - col.setURL(DetailsURL.fromString("plate-isPlated.api?sampleId=${" + Column.RowId.name() + "}")); - addColumn(col); - - addVocabularyDomains(); - - addColumn(Column.Properties); - - var colInputs = addColumn(Column.Inputs); - addMethod("Inputs", new LineageMethod(colInputs, true), Set.of(colInputs.getFieldKey())); - - var colOutputs = addColumn(Column.Outputs); - addMethod("Outputs", new LineageMethod(colOutputs, false), Set.of(colOutputs.getFieldKey())); - - addExpObjectMethod(); - - ActionURL detailsUrl = new ActionURL(ExperimentController.ShowMaterialAction.class, getContainer()); - DetailsURL url = new DetailsURL(detailsUrl, Collections.singletonMap("rowId", "RowId"), NullResult); - nameCol.setURL(url); - rowIdCol.setURL(url); - setDetailsURL(url); - - if (canUserAccessPhi()) - { - ActionURL updateActionURL = PageFlowUtil.urlProvider(ExperimentUrls.class).getUpdateMaterialQueryRowAction(getContainer(), this); - setUpdateURL(new DetailsURL(updateActionURL, Collections.singletonMap("RowId", "RowId"))); - - ActionURL insertActionURL = PageFlowUtil.urlProvider(ExperimentUrls.class).getInsertMaterialQueryRowAction(getContainer(), this); - setInsertURL(new DetailsURL(insertActionURL)); - } - else - { - setImportURL(LINK_DISABLER); - setInsertURL(LINK_DISABLER); - setUpdateURL(LINK_DISABLER); - } - - setTitleColumn(Column.Name.toString()); - - setDefaultVisibleColumns(defaultCols); - - MutableColumnInfo lineageLookup = ClosureQueryHelper.createAncestorLookupColumnInfo("Ancestors", this, _rootTable.getColumn("rowid"), _ss, true); - addColumn(lineageLookup); - } - - private ContainerFilter getSampleStatusLookupContainerFilter() - { - // The default lookup container filter is Current, but we want to have the default be CurrentPlusProjectAndShared - // for the sample status lookup since in the app project context we want to share status definitions across - // a given project instead of creating duplicate statuses in each subfolder project. - ContainerFilter.Type type = QueryService.get().getContainerFilterTypeForLookups(getContainer()); - type = type == null ? ContainerFilter.Type.CurrentPlusProjectAndShared : type; - return type.create(getUserSchema()); - } - - @Override - public Domain getDomain() - { - return getDomain(false); - } - - @Override - public Domain getDomain(boolean forUpdate) - { - return _ss == null ? null : _ss.getDomain(forUpdate); - } - - - public static String appendNameExpressionDescription(String currentDescription, String nameExpression, String nameExpressionPreview) - { - if (nameExpression == null) - return currentDescription; - - StringBuilder sb = new StringBuilder(); - if (currentDescription != null && !currentDescription.isEmpty()) - { - sb.append(currentDescription); - if (!currentDescription.endsWith(".")) - sb.append("."); - sb.append("\n"); - } - - sb.append("\nIf not provided, a unique name will be generated from the expression:\n"); - sb.append(nameExpression); - sb.append("."); - if (!StringUtils.isEmpty(nameExpressionPreview)) - { - sb.append("\nExample of name that will be generated from the current pattern: \n"); - sb.append(nameExpressionPreview); - } - - return sb.toString(); - } - - private void addSampleTypeColumns(ExpSampleType st, List visibleColumns) - { - TableInfo dbTable = ((ExpSampleTypeImpl)st).getTinfo(); - if (null == dbTable) - return; - - UserSchema schema = getUserSchema(); - Domain domain = st.getDomain(); - ColumnInfo rowIdColumn = getColumn(Column.RowId); - ColumnInfo lsidColumn = getColumn(Column.LSID); - ColumnInfo nameColumn = getColumn(Column.Name); - - visibleColumns.remove(FieldKey.fromParts(Column.Run.name())); - - // When not using name expressions, mark the ID columns as required. - // NOTE: If not explicitly set, the first domain property will be chosen as the ID column. - final List idCols = st.hasNameExpression() ? Collections.emptyList() : st.getIdCols(); - - Set mvColumns = domain.getProperties().stream() - .filter(ImportAliasable::isMvEnabled) - .map(dp -> FieldKey.fromParts(dp.getPropertyDescriptor().getMvIndicatorStorageColumnName())) - .collect(Collectors.toSet()); - - for (ColumnInfo dbColumn : dbTable.getColumns()) - { - // Don't include PHI columns in full text search index - // CONSIDER: Can we move this to a base class? Maybe in .addColumn() - if (schema.getUser().isSearchUser() && !dbColumn.getPHI().isLevelAllowed(PHI.NotPHI)) - continue; - - if ( - rowIdColumn.getFieldKey().equals(dbColumn.getFieldKey()) || - lsidColumn.getFieldKey().equals(dbColumn.getFieldKey()) || - nameColumn.getFieldKey().equals(dbColumn.getFieldKey()) - ) - { - continue; - } - - var wrapped = wrapColumnFromJoinedTable(dbColumn.getName(), dbColumn); - - // TODO missing values? comments? flags? - DomainProperty dp = domain.getPropertyByURI(dbColumn.getPropertyURI()); - var propColumn = copyColumnFromJoinedTable(null==dp ? dbColumn.getName() : dp.getName(), wrapped); - if (propColumn.getName().equalsIgnoreCase("genid")) - { - propColumn.setHidden(true); - propColumn.setUserEditable(false); - propColumn.setShownInDetailsView(false); - propColumn.setShownInInsertView(false); - propColumn.setShownInUpdateView(false); - } - if (null != dp) - { - PropertyColumn.copyAttributes(schema.getUser(), propColumn, dp.getPropertyDescriptor(), schema.getContainer(), - SchemaKey.fromParts("samples"), st.getName(), FieldKey.fromParts("RowId"), null, getLookupContainerFilter()); - - if (idCols.contains(dp)) - { - propColumn.setNullable(false); - propColumn.setDisplayColumnFactory(new IdColumnRendererFactory()); - } - - // Issue 38341: domain designer advanced settings 'show in default view' setting is not respected - if (!propColumn.isHidden()) - { - visibleColumns.add(propColumn.getFieldKey()); - } - - if (propColumn.isMvEnabled()) - { - // The column in the physical table has a "_MVIndicator" suffix, but we want to expose - // it with a "MVIndicator" suffix (no underscore) - var mvColumn = new AliasedColumn(this, dp.getName() + MvColumn.MV_INDICATOR_SUFFIX, - StorageProvisioner.get().getMvIndicatorColumn(dbTable, dp.getPropertyDescriptor(), "No MV column found for '" + dp.getName() + "' in sample type '" + getName() + "'")); - mvColumn.setLabel(dp.getLabel() != null ? dp.getLabel() : dp.getName() + " MV Indicator"); - mvColumn.setSqlTypeName("VARCHAR"); - mvColumn.setPropertyURI(dp.getPropertyURI()); - mvColumn.setNullable(true); - mvColumn.setUserEditable(false); - mvColumn.setHidden(true); - mvColumn.setMvIndicatorColumn(true); - - addColumn(mvColumn); - propColumn.setMvColumnName(FieldKey.fromParts(dp.getName() + MvColumn.MV_INDICATOR_SUFFIX)); - } - } - - if (!mvColumns.contains(propColumn.getFieldKey())) - addColumn(propColumn); - - } - - setDefaultVisibleColumns(visibleColumns); - } - - // These are mostly fields that are wrapped by fields with different names (see createColumn()) - // we could handle each case separately, but this is easier - static final Set wrappedFieldKeys = Set.of( - new FieldKey(null, "objectid"), - new FieldKey(null, "RowId"), - new FieldKey(null, "LSID"), // Flag - new FieldKey(null, "SourceApplicationId"), // SourceProtocolApplication - new FieldKey(null, "runId"), // Run, RunApplication - new FieldKey(null, "CpasType")); // SampleSet - static final Set ALL_COLUMNS = Set.of(); - - private @NotNull Set computeInnerSelectedColumns(Set selectedColumns) - { - if (null == selectedColumns) - return ALL_COLUMNS; - selectedColumns = new TreeSet<>(selectedColumns); - if (selectedColumns.contains(new FieldKey(null, StoredAmount))) - selectedColumns.add(new FieldKey(null, Units)); - if (selectedColumns.contains(new FieldKey(null, ExpMaterial.ALIQUOTED_FROM_INPUT))) - selectedColumns.add(new FieldKey(null, Column.AliquotedFromLSID.name())); - if (selectedColumns.contains(new FieldKey(null, Column.IsAliquot.name()))) - selectedColumns.add(new FieldKey(null, Column.RootMaterialRowId.name())); - selectedColumns.addAll(wrappedFieldKeys); - if (null != getFilter()) - selectedColumns.addAll(getFilter().getAllFieldKeys()); - return selectedColumns; - } - - @NotNull - @Override - public SQLFragment getFromSQL(String alias) - { - return getFromSQL(alias, null); - } - - @Override - public SQLFragment getFromSQLExpanded(String alias, Set selectedColumns) - { - SQLFragment sql = new SQLFragment("("); - boolean usedMaterialized; - - - // SELECT FROM - /* NOTE We want to avoid caching in paths where the table is actively being updated (e.g. loadRows) - * Unfortunately, we don't _really_ know when this is, but if we in a transaction that's a good guess. - * Also, we may use RemapCache for material lookup outside a transaction - */ - boolean onlyMaterialColums = false; - if (null != selectedColumns && !selectedColumns.isEmpty()) - onlyMaterialColums = selectedColumns.stream().allMatch(fk -> fk.getName().equalsIgnoreCase("Folder") || null != _rootTable.getColumn(fk)); - if (!onlyMaterialColums && null != _ss && null != _ss.getTinfo() && !getExpSchema().getDbSchema().getScope().isTransactionActive()) - { - sql.append(getMaterializedSQL()); - usedMaterialized = true; - } - else - { - sql.append(getJoinSQL(selectedColumns)); - usedMaterialized = false; - } - - // WHERE - SQLFragment filterFrag = getFilter().getSQLFragment(_rootTable, null); - sql.append("\n").append(filterFrag); - if (_ss != null && !usedMaterialized) - { - if (!filterFrag.isEmpty()) - sql.append(" AND "); - else - sql.append(" WHERE "); - sql.append("CpasType = ").appendValue(_ss.getLSID()); - } - sql.append(") ").appendIdentifier(alias); - - return getTransformedFromSQL(sql); - } - - @Override - public void setSupportTableRules(boolean b) - { - this._supportTableRules = b; - } - - @Override - public boolean supportTableRules() // intentional override - { - return _supportTableRules; - } - - @Override - protected @NotNull TableRules findTableRules() - { - Container definitionContainer = getUserSchema().getContainer(); - if (null != _ss) - definitionContainer = _ss.getContainer(); - return TableRulesManager.get().getTableRules(definitionContainer, getUserSchema().getUser(), getUserSchema().getContainer()); - } - - - static class InvalidationCounters - { - public final AtomicLong update, insert, delete, rollup; - InvalidationCounters() - { - long l = System.currentTimeMillis(); - update = new AtomicLong(l); - insert = new AtomicLong(l); - delete = new AtomicLong(l); - rollup = new AtomicLong(l); - } - } - - static final BlockingCache _materializedQueries = CacheManager.getBlockingStringKeyCache(CacheManager.UNLIMITED, CacheManager.HOUR, "materialized sample types", null); - static final Map _invalidationCounters = Collections.synchronizedMap(new HashMap<>()); - static final AtomicBoolean initializedListeners = new AtomicBoolean(false); - - // used by SampleTypeServiceImpl.refreshSampleTypeMaterializedView() - public static void refreshMaterializedView(final String lsid, SampleTypeServiceImpl.SampleChangeType reason) - { - var scope = ExperimentServiceImpl.getExpSchema().getScope(); - var runnable = new RefreshMaterializedViewRunnable(lsid, reason); - scope.addCommitTask(runnable, DbScope.CommitTaskOption.POSTCOMMIT); - } - - private static class RefreshMaterializedViewRunnable implements Runnable - { - private final String _lsid; - private final SampleTypeServiceImpl.SampleChangeType _reason; - - public RefreshMaterializedViewRunnable(String lsid, SampleTypeServiceImpl.SampleChangeType reason) - { - _lsid = lsid; - _reason = reason; - } - - @Override - public void run() - { - if (_reason == schema) - { - /* NOTE: MaterializedQueryHelper can detect data changes and refresh the materialized view using the provided SQL. - * It does not handle schema changes where the SQL itself needs to be updated. In this case, we remove the - * MQH from the cache to force the SQL to be regenerated. - */ - _materializedQueries.remove(_lsid); - return; - } - var counters = getInvalidateCounters(_lsid); - switch (_reason) - { - case insert -> counters.insert.incrementAndGet(); - case rollup -> counters.rollup.incrementAndGet(); - case update -> counters.update.incrementAndGet(); - case delete -> counters.delete.incrementAndGet(); - default -> throw new IllegalStateException("Unexpected value: " + _reason); - } - } - - @Override - public boolean equals(Object obj) - { - return obj instanceof RefreshMaterializedViewRunnable other && _lsid.equals(other._lsid) && _reason.equals(other._reason); - } - } - - private static InvalidationCounters getInvalidateCounters(String lsid) - { - if (!initializedListeners.getAndSet(true)) - { - CacheManager.addListener(_invalidationCounters::clear); - } - return _invalidationCounters.computeIfAbsent(lsid, (unused) -> - new InvalidationCounters() - ); - } - - /* SELECT and JOIN, does not include WHERE, same as getJoinSQL() */ - private SQLFragment getMaterializedSQL() - { - if (null == _ss) - return getJoinSQL(null); - - var mqh = _materializedQueries.get(_ss.getLSID(), null, (unusedKey, unusedArg) -> - { - /* NOTE: MaterializedQueryHelper does have a pattern to help with detecting schema changes. - * Previously it has been used on non-provisioned tables. It might be helpful to have a pattern, - * even if just to help with race-conditions. - * - * Maybe have a callback to generate the SQL dynamically, and verify that the sql is unchanged. - */ - SQLFragment viewSql = getJoinSQL(null).append(" WHERE CpasType = ").appendValue(_ss.getLSID()); - return (_MaterializedQueryHelper) new _MaterializedQueryHelper.Builder(_ss.getLSID(), "", getExpSchema().getDbSchema().getScope(), viewSql) - .addIndex("CREATE UNIQUE INDEX uq_${NAME}_rowid ON temp.${NAME} (rowid)") - .addIndex("CREATE UNIQUE INDEX uq_${NAME}_lsid ON temp.${NAME} (lsid)") - .addIndex("CREATE INDEX idx_${NAME}_container ON temp.${NAME} (container)") - .addIndex("CREATE INDEX idx_${NAME}_root ON temp.${NAME} (rootmaterialrowid)") - .addInvalidCheck(() -> String.valueOf(getInvalidateCounters(_ss.getLSID()).update.get())) - .build(); - }); - return new SQLFragment("SELECT * FROM ").append(mqh.getFromSql("_cached_view_")); - } - - - /** - * MaterializedQueryHelper has a built-in mechanism for tracking when a temp table needs to be recomputed. - * It does not help with incremental updates (except for providing the upsert() method). - * _MaterializedQueryHelper and _Materialized copy the pattern using class Invalidator. - */ - static class _MaterializedQueryHelper extends MaterializedQueryHelper - { - final String _lsid; - - static class Builder extends MaterializedQueryHelper.Builder - { - String _lsid; - - public Builder(String lsid, String prefix, DbScope scope, SQLFragment select) - { - super(prefix, scope, select); - this._lsid = lsid; - } - - @Override - public _MaterializedQueryHelper build() - { - return new _MaterializedQueryHelper(_lsid, _prefix, _scope, _select, _uptodate, _supplier, _indexes, _max, _isSelectInto); - } - } - - _MaterializedQueryHelper(String lsid, String prefix, DbScope scope, SQLFragment select, @Nullable SQLFragment uptodate, Supplier supplier, @Nullable Collection indexes, long maxTimeToCache, - boolean isSelectIntoSql) - { - super(prefix, scope, select, uptodate, supplier, indexes, maxTimeToCache, isSelectIntoSql); - this._lsid = lsid; - } - - @Override - protected Materialized createMaterialized(String txCacheKey) - { - DbSchema temp = DbSchema.getTemp(); - String name = _prefix + "_" + GUID.makeHash(); - _Materialized materialized = new _Materialized(this, name, txCacheKey, HeartBeat.currentTimeMillis(), "\"" + temp.getName() + "\".\"" + name + "\""); - initMaterialized(materialized); - return materialized; - } - - @Override - protected void incrementalUpdateBeforeSelect(Materialized m) - { - _Materialized materialized = (_Materialized) m; - - boolean lockAcquired = false; - try - { - lockAcquired = materialized.getLock().tryLock(1, TimeUnit.MINUTES); - if (Materialized.LoadingState.ERROR == materialized._loadingState.get()) - throw materialized._loadException; - - if (!materialized.incrementalDeleteCheck.stillValid(0)) - executeIncrementalDelete(); - if (!materialized.incrementalRollupCheck.stillValid(0)) - executeIncrementalRollup(); - if (!materialized.incrementalInsertCheck.stillValid(0)) - executeIncrementalInsert(); - } - catch (RuntimeException|InterruptedException ex) - { - RuntimeException rex = UnexpectedException.wrap(ex); - materialized.setError(rex); - // The only time I'd expect an error is due to a schema change race-condition, but that can happen in any code path. - - // Ensure that next refresh starts clean - _materializedQueries.remove(_lsid); - getInvalidateCounters(_lsid).update.incrementAndGet(); - throw rex; - } - finally - { - if (lockAcquired) - materialized.getLock().unlock(); - } - } - - void upsertWithRetry(SQLFragment sql) - { - // not actually read-only, but we don't want to start an explicit transaction - _scope.executeWithRetryReadOnly((tx) -> upsert(sql)); - } - - void executeIncrementalInsert() - { - SQLFragment incremental = new SQLFragment("INSERT INTO temp.${NAME}\n") - .append("SELECT * FROM (") - .append(getViewSourceSql()).append(") viewsource_\n") - .append("WHERE rowid > (SELECT COALESCE(MAX(rowid),0) FROM temp.${NAME})"); - upsertWithRetry(incremental); - } - - void executeIncrementalDelete() - { - var d = CoreSchema.getInstance().getSchema().getSqlDialect(); - // POSTGRES bug??? the obvious query is _very_ slow O(n^2) - // DELETE FROM temp.${NAME} WHERE rowid NOT IN (SELECT rowid FROM exp.material WHERE cpastype = <<_lsid>>) - SQLFragment incremental = new SQLFragment() - .append("WITH deleted AS (SELECT rowid FROM temp.${NAME} EXCEPT SELECT rowid FROM exp.material WHERE cpastype = ").appendValue(_lsid,d).append(")\n") - .append("DELETE FROM temp.${NAME} WHERE rowid IN (SELECT rowid from deleted)\n"); - upsertWithRetry(incremental); - } - - void executeIncrementalRollup() - { - var d = CoreSchema.getInstance().getSchema().getSqlDialect(); - SQLFragment incremental = new SQLFragment(); - if (d.isPostgreSQL()) - { - incremental - .append("UPDATE temp.${NAME} AS st\n") - .append("SET aliquotcount = expm.aliquotcount, availablealiquotcount = expm.availablealiquotcount, aliquotvolume = expm.aliquotvolume, availablealiquotvolume = expm.availablealiquotvolume, aliquotunit = expm.aliquotunit\n") - .append("FROM exp.Material AS expm\n") - .append("WHERE expm.rowid = st.rowid AND expm.cpastype = ").appendValue(_lsid,d).append(" AND (\n") - .append(" st.aliquotcount IS DISTINCT FROM expm.aliquotcount OR ") - .append(" st.availablealiquotcount IS DISTINCT FROM expm.availablealiquotcount OR ") - .append(" st.aliquotvolume IS DISTINCT FROM expm.aliquotvolume OR ") - .append(" st.availablealiquotvolume IS DISTINCT FROM expm.availablealiquotvolume OR ") - .append(" st.aliquotunit IS DISTINCT FROM expm.aliquotunit") - .append(")"); - } - else - { - // SQL Server 2022 supports IS DISTINCT FROM - incremental - .append("UPDATE st\n") - .append("SET aliquotcount = expm.aliquotcount, availablealiquotcount = expm.availablealiquotcount, aliquotvolume = expm.aliquotvolume, availablealiquotvolume = expm.availablealiquotvolume, aliquotunit = expm.aliquotunit\n") - .append("FROM temp.${NAME} st, exp.Material expm\n") - .append("WHERE expm.rowid = st.rowid AND expm.cpastype = ").appendValue(_lsid,d).append(" AND (\n") - .append(" COALESCE(st.aliquotcount,-2147483648) <> COALESCE(expm.aliquotcount,-2147483648) OR ") - .append(" COALESCE(st.availablealiquotcount,-2147483648) <> COALESCE(expm.availablealiquotcount,-2147483648) OR ") - .append(" COALESCE(st.aliquotvolume,-2147483648) <> COALESCE(expm.aliquotvolume,-2147483648) OR ") - .append(" COALESCE(st.availablealiquotvolume,-2147483648) <> COALESCE(expm.availablealiquotvolume,-2147483648) OR ") - .append(" COALESCE(st.aliquotunit,'-') <> COALESCE(expm.aliquotunit,'-')") - .append(")"); - } - upsertWithRetry(incremental); - } - } - - static class _Materialized extends MaterializedQueryHelper.Materialized - { - final MaterializedQueryHelper.Invalidator incrementalInsertCheck; - final MaterializedQueryHelper.Invalidator incrementalRollupCheck; - final MaterializedQueryHelper.Invalidator incrementalDeleteCheck; - - _Materialized(_MaterializedQueryHelper mqh, String tableName, String cacheKey, long created, String sql) - { - super(mqh, tableName, cacheKey, created, sql); - final InvalidationCounters counters = getInvalidateCounters(mqh._lsid); - incrementalInsertCheck = new MaterializedQueryHelper.SupplierInvalidator(() -> String.valueOf(counters.insert.get())); - incrementalRollupCheck = new MaterializedQueryHelper.SupplierInvalidator(() -> String.valueOf(counters.rollup.get())); - incrementalDeleteCheck = new MaterializedQueryHelper.SupplierInvalidator(() -> String.valueOf(counters.delete.get())); - } - - @Override - public void reset() - { - super.reset(); - long now = HeartBeat.currentTimeMillis(); - incrementalInsertCheck.stillValid(now); - incrementalRollupCheck.stillValid(now); - incrementalDeleteCheck.stillValid(now); - } - - Lock getLock() - { - return _loadingLock; - } - } - - - /* SELECT and JOIN, does not include WHERE */ - private SQLFragment getJoinSQL(Set selectedColumns) - { - TableInfo provisioned = null == _ss ? null : _ss.getTinfo(); - Set provisionedCols = new CaseInsensitiveHashSet(provisioned != null ? provisioned.getColumnNameSet() : Collections.emptySet()); - provisionedCols.remove(Column.RowId.name()); - provisionedCols.remove(Column.LSID.name()); - provisionedCols.remove(Column.Name.name()); - boolean hasProvisionedColumns = containsProvisionedColumns(selectedColumns, provisionedCols); - - boolean hasSampleColumns = false; - boolean hasAliquotColumns = false; - - Set materialCols = new CaseInsensitiveHashSet(_rootTable.getColumnNameSet()); - selectedColumns = computeInnerSelectedColumns(selectedColumns); - - SQLFragment sql = new SQLFragment(); - sql.appendComment("", getSqlDialect()); - sql.append("SELECT "); - String comma = ""; - for (String materialCol : materialCols) - { - // don't need to generate SQL for columns that aren't selected - if (ALL_COLUMNS == selectedColumns || selectedColumns.contains(new FieldKey(null, materialCol))) - { - sql.append(comma).append("m.").appendIdentifier(materialCol); - comma = ", "; - } - } - if (null != provisioned && hasProvisionedColumns) - { - for (ColumnInfo propertyColumn : provisioned.getColumns()) - { - // don't select twice - if ( - Column.RowId.name().equalsIgnoreCase(propertyColumn.getColumnName()) || - Column.LSID.name().equalsIgnoreCase(propertyColumn.getColumnName()) || - Column.Name.name().equalsIgnoreCase(propertyColumn.getColumnName()) - ) - { - continue; - } - - // don't need to generate SQL for columns that aren't selected - if (ALL_COLUMNS == selectedColumns || selectedColumns.contains(propertyColumn.getFieldKey()) || propertyColumn.isMvIndicatorColumn()) - { - sql.append(comma); - boolean rootField = StringUtils.isEmpty(propertyColumn.getDerivationDataScope()) - || ExpSchema.DerivationDataScopeType.ParentOnly.name().equalsIgnoreCase(propertyColumn.getDerivationDataScope()); - if ("genid".equalsIgnoreCase(propertyColumn.getColumnName()) || propertyColumn.isUniqueIdField()) - { - sql.append(propertyColumn.getValueSql("m_aliquot")).append(" AS ").appendIdentifier(propertyColumn.getSelectIdentifier()); - hasAliquotColumns = true; - } - else if (rootField) - { - sql.append(propertyColumn.getValueSql("m_sample")).append(" AS ").appendIdentifier(propertyColumn.getSelectIdentifier()); - hasSampleColumns = true; - } - else - { - sql.append(propertyColumn.getValueSql("m_aliquot")).append(" AS ").appendIdentifier(propertyColumn.getSelectIdentifier()); - hasAliquotColumns = true; - } - comma = ", "; - } - } - } - - sql.append("\nFROM "); - sql.append(_rootTable, "m"); - if (hasSampleColumns) - sql.append(" INNER JOIN ").append(provisioned, "m_sample").append(" ON m.RootMaterialRowId = m_sample.RowId"); - if (hasAliquotColumns) - sql.append(" INNER JOIN ").append(provisioned, "m_aliquot").append(" ON m.RowId = m_aliquot.RowId"); - - sql.appendComment("", getSqlDialect()); - return sql; - } - - private class IdColumnRendererFactory implements DisplayColumnFactory - { - @Override - public DisplayColumn createRenderer(ColumnInfo colInfo) - { - return new IdColumnRenderer(colInfo); - } - } - - private static class IdColumnRenderer extends DataColumn - { - public IdColumnRenderer(ColumnInfo col) - { - super(col); - } - - @Override - protected boolean isDisabledInput(RenderContext ctx) - { - return !super.isDisabledInput() && ctx.getMode() != DataRegion.MODE_INSERT; - } - } - - private static class SampleTypeAmountDisplayColumn extends ExprColumn - { - public SampleTypeAmountDisplayColumn(TableInfo parent, String amountFieldName, String unitFieldName, String label, Set importAliases, Unit typeUnit) - { - super(parent, FieldKey.fromParts(amountFieldName), new SQLFragment( - "(CASE WHEN ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(unitFieldName) - .append(" = ? AND ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(amountFieldName) - .append(" IS NOT NULL THEN CAST(").append(ExprColumn.STR_TABLE_ALIAS + ".").append(amountFieldName) - .append(" / ? AS ") - .append(parent.getSqlDialect().isPostgreSQL() ? "DECIMAL" : "DOUBLE PRECISION") - .append(") ELSE ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(amountFieldName) - .append(" END)") - .add(typeUnit.getBase().toString()) - .add(typeUnit.getValue()), - JdbcType.DOUBLE); - - setLabel(label); - setImportAliasesSet(importAliases); - } - } - - private static class SampleTypeUnitDisplayColumn extends ExprColumn - { - public SampleTypeUnitDisplayColumn(TableInfo parent, String unitFieldName, Unit typeUnit) - { - super(parent, FieldKey.fromParts(Column.Units.name()), new SQLFragment( - "(CASE WHEN ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(unitFieldName) - .append(" = ? THEN ? ELSE ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(unitFieldName) - .append(" END)") - .add(typeUnit.getBase().toString()) - .add(typeUnit.toString()), - JdbcType.VARCHAR); - } - } - - @Override - public QueryUpdateService getUpdateService() - { - return new SampleTypeUpdateServiceDI(this, _ss); - } - - @Override - public boolean hasPermission(@NotNull UserPrincipal user, @NotNull Class perm) - { - if (_ss == null) - { - // Allow read and delete for exp.Materials. - // Don't allow insert/update on exp.Materials without a sample type. - if (perm == DeletePermission.class || perm == ReadPermission.class) - return getContainer().hasPermission(user, perm); - return false; - } - - if (_ss.isMedia() && perm == ReadPermission.class) - return getContainer().hasPermission(user, MediaReadPermission.class); - - return super.hasPermission(user, perm); - } - - @NotNull - @Override - public List getUniqueIndices() - { - // Rewrite the "idx_material_ak" unique index over "Folder", "SampleSet", "Name" to just "Name" - // Issue 25397: Don't include the "idx_material_ak" index if the "Name" column hasn't been added to the table. - // Some FKs to ExpMaterialTable don't include the "Name" column (e.g. NabBaseTable.Specimen) - String indexName = "idx_material_ak"; - List ret = new ArrayList<>(super.getUniqueIndices()); - if (getColumn("Name") != null) - ret.add(new IndexDefinition(indexName, IndexType.Unique, Arrays.asList(getColumn("Name")), null)); - else - ret.removeIf( def -> def.name().equals(indexName)); - return Collections.unmodifiableList(ret); - } - - - // - // UpdatableTableInfo - // - - - @Override - public @Nullable Long getOwnerObjectId() - { - return OntologyManager.ensureObject(_ss.getContainer(), _ss.getLSID(), (Long) null); - } - - @Nullable - @Override - public CaseInsensitiveHashMap remapSchemaColumns() - { - CaseInsensitiveHashMap m = new CaseInsensitiveHashMap<>(); - - if (null != getRealTable().getColumn("container") && null != getColumn("folder")) - { - m.put("container", "folder"); - } - - for (ColumnInfo col : getColumns()) - { - if (col.getMvColumnName() != null) - m.put(col.getName() + "_" + MvColumn.MV_INDICATOR_SUFFIX, col.getMvColumnName().getName()); - } - - return m; - } - - @Override - public Set getAltMergeKeys(DataIteratorContext context) - { - if (context.getInsertOption().updateOnly && context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate)) - return getAltKeysForUpdate(); - - return MATERIAL_ALT_MERGE_KEYS; - } - - @NotNull - @Override - public Set getAltKeysForUpdate() - { - return MATERIAL_ALT_UPDATE_KEYS; - } - - @Override - @NotNull - public List> getAdditionalRequiredInsertColumns() - { - if (getSampleType() == null) - return Collections.emptyList(); - - try - { - return getRequiredParentImportFields(getSampleType().getRequiredImportAliases()); - } - catch (IOException e) - { - return Collections.emptyList(); - } - } - - @Override - public DataIteratorBuilder persistRows(DataIteratorBuilder data, DataIteratorContext context) - { - TableInfo propertiesTable = _ss.getTinfo(); - - // The specimens sample type doesn't have a properties table - if (propertiesTable == null) - { - return data; - } - - long sampleTypeObjectId = requireNonNull(getOwnerObjectId()); - - // TODO: subclass PersistDataIteratorBuilder to index Materials! not DataClass! - try - { - var persist = new ExpDataIterators.PersistDataIteratorBuilder(data, this, propertiesTable, _ss, getUserSchema().getContainer(), getUserSchema().getUser(), _ss.getImportAliases(), sampleTypeObjectId) - .setFileLinkDirectory(SAMPLETYPE_FILE_DIRECTORY); - ExperimentServiceImpl experimentServiceImpl = ExperimentServiceImpl.get(); - SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(getContainer(), SearchService.PRIORITY.modified); - - persist.setIndexFunction(searchIndexDataKeys -> propertiesTable.getSchema().getScope().addCommitTask(() -> - { - List lsids = searchIndexDataKeys.lsids(); - List orderedRowIds = searchIndexDataKeys.orderedRowIds(); - - // Issue 51263: order by RowId to reduce deadlock - ListUtils.partition(orderedRowIds, 100).forEach(sublist -> - queue.addRunnable((q) -> - { - for (ExpMaterialImpl expMaterial : experimentServiceImpl.getExpMaterials(sublist)) - expMaterial.index(q, this); - }) - ); - - ListUtils.partition(lsids, 100).forEach(sublist -> - queue.addRunnable((q) -> - { - for (ExpMaterialImpl expMaterial : experimentServiceImpl.getExpMaterialsByLsid(sublist)) - expMaterial.index(q, this); - }) - ); - }, DbScope.CommitTaskOption.POSTCOMMIT) - ); - - DataIteratorBuilder builder = LoggingDataIterator.wrap(persist); - return LoggingDataIterator.wrap(new AliasDataIteratorBuilder(builder, getUserSchema().getContainer(), getUserSchema().getUser(), ExperimentService.get().getTinfoMaterialAliasMap(), _ss, true)); - } - catch (IOException e) - { - throw new UncheckedIOException(e); - } - } - - @Override - @NotNull - public AuditBehaviorType getDefaultAuditBehavior() - { - return AuditBehaviorType.DETAILED; - } - - static final Set excludeFromDetailedAuditField; - static - { - var set = new CaseInsensitiveHashSet(); - set.addAll(TableInfo.defaultExcludedDetailedUpdateAuditFields); - set.addAll(ExpDataIterators.NOT_FOR_UPDATE); - // We don't want the inventory columns to show up in the sample timeline audit record; - // they are captured in their own audit record. - set.addAll(InventoryService.InventoryStatusColumn.names()); - excludeFromDetailedAuditField = Collections.unmodifiableSet(set); - } - - @Override - public @NotNull Set getExcludedDetailedUpdateAuditFields() - { - // uniqueId fields don't change in reality, so exclude them from the audit updates - Set excluded = new CaseInsensitiveHashSet(); - excluded.addAll(this.getUniqueIdFields()); - excluded.addAll(excludeFromDetailedAuditField); - return excluded; - } - - @Override - public List> getImportTemplates(ViewContext ctx) - { - // respect any metadata overrides - if (getRawImportTemplates() != null) - return super.getImportTemplates(ctx); - - List> templates = new ArrayList<>(); - ActionURL url = PageFlowUtil.urlProvider(QueryUrls.class).urlCreateExcelTemplate(ctx.getContainer(), getPublicSchemaName(), getName()); - url.addParameter("headerType", ColumnHeaderType.ImportField.name()); - try - { - if (getSampleType() != null && !getSampleType().getImportAliases().isEmpty()) - { - for (String aliasKey : getSampleType().getImportAliases().keySet()) - url.addParameter("includeColumn", aliasKey); - } - } - catch (IOException e) - {} - templates.add(Pair.of("Download Template", url.toString())); - return templates; - } - - @Override - public void overlayMetadata(String tableName, UserSchema schema, Collection errors) - { - if (SamplesSchema.SCHEMA_NAME.equals(schema.getName())) - { - Collection metadata = QueryService.get().findMetadataOverride(schema, SamplesSchema.SCHEMA_METADATA_NAME, false, false, errors, null); - if (null != metadata) - { - overlayMetadata(metadata, schema, errors); - } - } - super.overlayMetadata(tableName, schema, errors); - } - - static class SampleTypeAmountPrecisionDisplayColumn extends DataColumn - { - Unit typeUnit; - boolean applySampleTypePrecision = true; - - public SampleTypeAmountPrecisionDisplayColumn(ColumnInfo col, Unit typeUnit) { - super(col, false); - this.typeUnit = typeUnit; - this.applySampleTypePrecision = col.getFormat() == null; // only apply if no custom format is set by user - } - - @Override - public Object getDisplayValue(RenderContext ctx) - { - Object value = super.getDisplayValue(ctx); - if (this.applySampleTypePrecision && value != null) - { - int scale = this.typeUnit == null ? Quantity.DEFAULT_PRECISION_SCALE : this.typeUnit.getPrecisionScale(); - value = Precision.round(Double.valueOf(value.toString()), scale); - } - return value; - } - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.experiment.api; + +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.math3.util.Precision; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.assay.plate.AssayPlateMetadataService; +import org.labkey.api.audit.AuditHandler; +import org.labkey.api.cache.BlockingCache; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.compliance.TableRules; +import org.labkey.api.compliance.TableRulesManager; +import org.labkey.api.data.ColumnHeaderType; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DataColumn; +import org.labkey.api.data.DataRegion; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.DisplayColumn; +import org.labkey.api.data.DisplayColumnFactory; +import org.labkey.api.data.ForeignKey; +import org.labkey.api.data.ImportAliasable; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.MaterializedQueryHelper; +import org.labkey.api.data.MutableColumnInfo; +import org.labkey.api.data.PHI; +import org.labkey.api.data.RenderContext; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.Sort; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.UnionContainerFilter; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.LoggingDataIterator; +import org.labkey.api.dataiterator.SimpleTranslator; +import org.labkey.api.exp.MvColumn; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyColumn; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.ExperimentUrls; +import org.labkey.api.exp.api.NameExpressionOptionService; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.DomainUtil; +import org.labkey.api.exp.query.ExpDataTable; +import org.labkey.api.exp.query.ExpMaterialTable; +import org.labkey.api.exp.query.ExpSampleTypeTable; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.exp.query.SamplesSchema; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.inventory.InventoryService; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; +import org.labkey.api.qc.SampleStatusService; +import org.labkey.api.query.AliasedColumn; +import org.labkey.api.query.DetailsURL; +import org.labkey.api.query.ExprColumn; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.LookupForeignKey; +import org.labkey.api.query.QueryException; +import org.labkey.api.query.QueryForeignKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateService; +import org.labkey.api.query.QueryUrls; +import org.labkey.api.query.RowIdForeignKey; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.column.BuiltInColumnTypes; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.UserPrincipal; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.InsertPermission; +import org.labkey.api.security.permissions.MediaReadPermission; +import org.labkey.api.security.permissions.MoveEntitiesPermission; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.util.GUID; +import org.labkey.api.util.HeartBeat; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StringExpression; +import org.labkey.api.util.UnexpectedException; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.ViewContext; +import org.labkey.data.xml.TableType; +import org.labkey.experiment.ExpDataIterators; +import org.labkey.experiment.ExpDataIterators.AliasDataIteratorBuilder; +import org.labkey.experiment.controllers.exp.ExperimentController; +import org.labkey.experiment.lineage.LineageMethod; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static java.util.Objects.requireNonNull; +import static org.labkey.api.exp.api.SampleTypeDomainKind.ALIQUOT_COUNT_LABEL; +import static org.labkey.api.exp.api.SampleTypeDomainKind.ALIQUOT_VOLUME_LABEL; +import static org.labkey.api.exp.api.SampleTypeDomainKind.AVAILABLE_ALIQUOT_COUNT_LABEL; +import static org.labkey.api.exp.api.SampleTypeDomainKind.AVAILABLE_ALIQUOT_VOLUME_LABEL; +import static org.labkey.api.exp.api.SampleTypeDomainKind.SAMPLETYPE_FILE_DIRECTORY; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotCount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotVolume; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.AvailableAliquotCount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.AvailableAliquotVolume; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; +import static org.labkey.api.util.StringExpressionFactory.AbstractStringExpression.NullValueBehavior.NullResult; +import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.schema; + +public class ExpMaterialTableImpl extends ExpRunItemTableImpl implements ExpMaterialTable +{ + ExpSampleTypeImpl _ss; + Set _uniqueIdFields; + boolean _supportTableRules = true; + + public static final Set MATERIAL_ALT_MERGE_KEYS; + public static final Set MATERIAL_ALT_UPDATE_KEYS; + static { + MATERIAL_ALT_MERGE_KEYS = Set.of(Column.MaterialSourceId.name(), Column.Name.name()); + MATERIAL_ALT_UPDATE_KEYS = Set.of(Column.LSID.name()); + } + + public ExpMaterialTableImpl(UserSchema schema, ContainerFilter cf, @Nullable ExpSampleType sampleType) + { + super(ExpSchema.TableType.Materials.name(), ExperimentServiceImpl.get().getTinfoMaterial(), schema, cf); + setDetailsURL(new DetailsURL(new ActionURL(ExperimentController.ShowMaterialAction.class, schema.getContainer()), Collections.singletonMap("rowId", "rowId"), NullResult)); + setPublicSchemaName(ExpSchema.SCHEMA_NAME); + addAllowablePermission(InsertPermission.class); + addAllowablePermission(UpdatePermission.class); + addAllowablePermission(MoveEntitiesPermission.class); + setAllowedInsertOption(QueryUpdateService.InsertOption.MERGE); + setSampleType(sampleType); + } + + public Set getUniqueIdFields() + { + if (_uniqueIdFields == null) + { + _uniqueIdFields = new CaseInsensitiveHashSet(); + _uniqueIdFields.addAll(getColumns().stream().filter(ColumnInfo::isUniqueIdField).map(ColumnInfo::getName).collect(Collectors.toSet())); + } + return _uniqueIdFields; + } + + @Override + protected ColumnInfo resolveColumn(String name) + { + ColumnInfo result = super.resolveColumn(name); + if (result == null) + { + if ("CpasType".equalsIgnoreCase(name)) + result = createColumn(Column.SampleSet.name(), Column.SampleSet); + else if (Column.Property.name().equalsIgnoreCase(name)) + result = createPropertyColumn(Column.Property.name()); + else if (Column.QueryableInputs.name().equalsIgnoreCase(name)) + result = createColumn(Column.QueryableInputs.name(), Column.QueryableInputs); + } + return result; + } + + @Override + public ColumnInfo getExpObjectColumn() + { + var ret = wrapColumn("ExpMaterialTableImpl_object_", _rootTable.getColumn("objectid")); + ret.setConceptURI(BuiltInColumnTypes.EXPOBJECTID_CONCEPT_URI); + return ret; + } + + @Override + public AuditHandler getAuditHandler(AuditBehaviorType auditBehaviorType) + { + if (getUserSchema().getName().equalsIgnoreCase(SamplesSchema.SCHEMA_NAME)) + { + // Special case sample auditing to help build a useful timeline view + return SampleTypeServiceImpl.get(); + } + + return super.getAuditHandler(auditBehaviorType); + } + + @Override + public MutableColumnInfo createColumn(String alias, Column column) + { + switch (column) + { + case Folder -> + { + return wrapColumn(alias, _rootTable.getColumn("Container")); + } + case LSID -> + { + return wrapColumn(alias, _rootTable.getColumn(Column.LSID.name())); + } + case MaterialSourceId -> + { + var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.MaterialSourceId.name())); + columnInfo.setFk(new LookupForeignKey(getLookupContainerFilter(), null, null, null, null, "RowId", "Name") + { + @Override + public TableInfo getLookupTableInfo() + { + ExpSampleTypeTable sampleTypeTable = ExperimentService.get().createSampleTypeTable(ExpSchema.TableType.SampleSets.toString(), _userSchema, getLookupContainerFilter()); + sampleTypeTable.populate(); + return sampleTypeTable; + } + + @Override + public StringExpression getURL(ColumnInfo parent) + { + return super.getURL(parent, true); + } + }); + columnInfo.setUserEditable(false); + columnInfo.setReadOnly(true); + columnInfo.setHidden(true); + return columnInfo; + } + case RootMaterialRowId -> + { + var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.RootMaterialRowId.name())); + columnInfo.setFk(getExpSchema().getMaterialForeignKey(getLookupContainerFilter(), Column.RowId.name())); + columnInfo.setLabel("Root Material"); + columnInfo.setUserEditable(false); + + // NK: Here we mark the column as not required AND nullable which is the opposite of the database where + // a NOT NULL constraint is in place. This is done to avoid the RequiredValidator check upon updating a row. + // See ExpMaterialValidatorIterator. + columnInfo.setRequired(false); + columnInfo.setNullable(true); + + return columnInfo; + } + case AliquotedFromLSID -> + { + var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.AliquotedFromLSID.name())); + columnInfo.setSqlTypeName("lsidtype"); + columnInfo.setFk(getExpSchema().getMaterialForeignKey(getLookupContainerFilter(), Column.LSID.name())); + columnInfo.setLabel("Aliquoted From Parent"); + return columnInfo; + } + case IsAliquot -> + { + String rootMaterialRowIdField = ExprColumn.STR_TABLE_ALIAS + "." + Column.RootMaterialRowId.name(); + String rowIdField = ExprColumn.STR_TABLE_ALIAS + "." + Column.RowId.name(); + ExprColumn columnInfo = new ExprColumn(this, FieldKey.fromParts(Column.IsAliquot.name()), new SQLFragment( + "(CASE WHEN (" + rootMaterialRowIdField + " = " + rowIdField + ") THEN ").append(getSqlDialect().getBooleanFALSE()) + .append(" WHEN ").append(rowIdField).append(" IS NOT NULL THEN ").append(getSqlDialect().getBooleanTRUE()) // Issue 52745 + .append(" ELSE NULL END)"), JdbcType.BOOLEAN); + columnInfo.setLabel("Is Aliquot"); + columnInfo.setDescription("Identifies if the material is a sample or an aliquot"); + columnInfo.setUserEditable(false); + columnInfo.setReadOnly(true); + columnInfo.setHidden(false); + return columnInfo; + } + case Name -> + { + var nameCol = wrapColumn(alias, _rootTable.getColumn(column.toString())); + // shut off this field in insert and update views if user specified names are not allowed + if (!NameExpressionOptionService.get().getAllowUserSpecificNamesValue(getContainer())) + { + nameCol.setShownInInsertView(false); + nameCol.setShownInUpdateView(false); + } + return nameCol; + } + case RawAmount -> + { + var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.StoredAmount.name())); + columnInfo.setDisplayColumnFactory(colInfo -> new SampleTypeAmountPrecisionDisplayColumn(colInfo, null)); + columnInfo.setDescription("The amount of this sample, in the base unit for the sample type's display unit (if defined), currently on hand."); + columnInfo.setUserEditable(false); + columnInfo.setReadOnly(true); + return columnInfo; + } + case StoredAmount -> + { + String label = StoredAmount.label(); + Set importAliases = Set.of(label, "Stored Amount"); + Unit typeUnit = getSampleTypeUnit(); + if (typeUnit != null) + { + SampleTypeAmountDisplayColumn columnInfo = new SampleTypeAmountDisplayColumn(this, Column.StoredAmount.name(), Column.Units.name(), label, importAliases, typeUnit); + columnInfo.setDisplayColumnFactory(colInfo -> new SampleTypeAmountPrecisionDisplayColumn(colInfo, typeUnit)); + columnInfo.setDescription("The amount of this sample, in the display unit for the sample type, currently on hand."); + columnInfo.setShownInUpdateView(true); + columnInfo.setShownInInsertView(true); + columnInfo.setUserEditable(true); + columnInfo.setCalculated(false); + return columnInfo; + } + else + { + var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.StoredAmount.name())); + columnInfo.setDisplayColumnFactory(colInfo -> new SampleTypeAmountPrecisionDisplayColumn(colInfo, null)); + columnInfo.setLabel(label); + columnInfo.setImportAliasesSet(importAliases); + columnInfo.setDescription("The amount of this sample currently on hand."); + return columnInfo; + } + } + case RawUnits -> + { + var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.Units.name())); + columnInfo.setDescription("The units associated with the Stored Amount for this sample."); + columnInfo.setUserEditable(false); + columnInfo.setReadOnly(true); + return columnInfo; + } + case Units -> + { + ForeignKey fk = new LookupForeignKey("Value", "Value") + { + @Override + public @Nullable TableInfo getLookupTableInfo() + { + return getExpSchema().getTable(ExpSchema.MEASUREMENT_UNITS_TABLE); + } + }; + + Unit typeUnit = getSampleTypeUnit(); + if (typeUnit != null) + { + SampleTypeUnitDisplayColumn columnInfo = new SampleTypeUnitDisplayColumn(this, Column.Units.name(), typeUnit); + columnInfo.setFk(fk); + columnInfo.setDescription("The sample type display units associated with the Amount for this sample."); + columnInfo.setShownInUpdateView(true); + columnInfo.setShownInInsertView(true); + columnInfo.setUserEditable(true); + columnInfo.setCalculated(false); + return columnInfo; + } + else + { + var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.Units.name())); + columnInfo.setFk(fk); + columnInfo.setDescription("The units associated with the Stored Amount for this sample."); + return columnInfo; + } + } + case Description -> + { + return wrapColumn(alias, _rootTable.getColumn(Column.Description.name())); + } + case SampleSet -> + { + var columnInfo = wrapColumn(alias, _rootTable.getColumn("CpasType")); + // NOTE: populateColumns() overwrites this with a QueryForeignKey. Can this be removed? + columnInfo.setFk(new LookupForeignKey(getContainerFilter(), null, null, null, null, "LSID", "Name") + { + @Override + public TableInfo getLookupTableInfo() + { + ExpSampleTypeTable sampleTypeTable = ExperimentService.get().createSampleTypeTable(ExpSchema.TableType.SampleSets.toString(), _userSchema, getLookupContainerFilter()); + sampleTypeTable.populate(); + return sampleTypeTable; + } + + @Override + public StringExpression getURL(ColumnInfo parent) + { + return super.getURL(parent, true); + } + }); + return columnInfo; + } + case SourceProtocolLSID -> + { + // NOTE: This column is incorrectly named "Protocol", but we are keeping it for backwards compatibility to avoid breaking queries in hvtnFlow module + ExprColumn columnInfo = new ExprColumn(this, ExpDataTable.Column.Protocol.toString(), new SQLFragment( + "(SELECT ProtocolLSID FROM " + ExperimentServiceImpl.get().getTinfoProtocolApplication() + " pa " + + " WHERE pa.RowId = " + ExprColumn.STR_TABLE_ALIAS + ".SourceApplicationId)"), JdbcType.VARCHAR); + columnInfo.setSqlTypeName("lsidtype"); + columnInfo.setFk(getExpSchema().getProtocolForeignKey(getContainerFilter(), "LSID")); + columnInfo.setLabel("Source Protocol"); + columnInfo.setDescription("Contains a reference to the protocol for the protocol application that created this sample"); + columnInfo.setUserEditable(false); + columnInfo.setReadOnly(true); + columnInfo.setHidden(true); + return columnInfo; + } + case SourceProtocolApplication -> + { + var columnInfo = wrapColumn(alias, _rootTable.getColumn("SourceApplicationId")); + columnInfo.setFk(getExpSchema().getProtocolApplicationForeignKey(getContainerFilter())); + columnInfo.setUserEditable(false); + columnInfo.setReadOnly(true); + columnInfo.setHidden(true); + columnInfo.setAutoIncrement(false); + return columnInfo; + } + case SourceApplicationInput -> + { + var col = createEdgeColumn(alias, Column.SourceProtocolApplication, ExpSchema.TableType.MaterialInputs); + col.setDescription("Contains a reference to the MaterialInput row between this ExpMaterial and it's SourceProtocolApplication"); + col.setHidden(true); + return col; + } + case RunApplication -> + { + SQLFragment sql = new SQLFragment("(SELECT pa.rowId FROM ") + .append(ExperimentService.get().getTinfoProtocolApplication(), "pa") + .append(" WHERE pa.runId = ").append(ExprColumn.STR_TABLE_ALIAS).append(".runId") + .append(" AND pa.cpasType = ").appendValue(ExpProtocol.ApplicationType.ExperimentRunOutput) + .append(")"); + + var col = new ExprColumn(this, alias, sql, JdbcType.INTEGER); + col.setFk(getExpSchema().getProtocolApplicationForeignKey(getContainerFilter())); + col.setDescription("Contains a reference to the ExperimentRunOutput protocol application of the run that created this sample"); + col.setUserEditable(false); + col.setReadOnly(true); + col.setHidden(true); + return col; + } + case RunApplicationOutput -> + { + var col = createEdgeColumn(alias, Column.RunApplication, ExpSchema.TableType.MaterialInputs); + col.setDescription("Contains a reference to the MaterialInput row between this ExpMaterial and it's RunOutputApplication"); + return col; + } + case Run -> + { + var ret = wrapColumn(alias, _rootTable.getColumn("RunId")); + ret.setReadOnly(true); + return ret; + } + case RowId -> + { + var ret = wrapColumn(alias, _rootTable.getColumn("RowId")); + // When no sorts are added by views, QueryServiceImpl.createDefaultSort() adds the primary key's default sort direction + ret.setSortDirection(Sort.SortDirection.DESC); + ret.setFk(new RowIdForeignKey(ret)); + ret.setUserEditable(false); + ret.setHidden(true); + ret.setShownInInsertView(false); + ret.setHasDbSequence(true); + ret.setIsRootDbSequence(true); + return ret; + } + case Property -> + { + return createPropertyColumn(alias); + } + case Flag -> + { + return createFlagColumn(alias); + } + case Created -> + { + return wrapColumn(alias, _rootTable.getColumn("Created")); + } + case CreatedBy -> + { + return createUserColumn(alias, _rootTable.getColumn("CreatedBy")); + } + case Modified -> + { + return wrapColumn(alias, _rootTable.getColumn("Modified")); + } + case ModifiedBy -> + { + return createUserColumn(alias, _rootTable.getColumn("ModifiedBy")); + } + case Alias -> + { + return createAliasColumn(alias, ExperimentService.get()::getTinfoMaterialAliasMap); + } + case Inputs -> + { + return createLineageColumn(this, alias, true, false); + } + case QueryableInputs -> + { + return createLineageColumn(this, alias, true, true); + } + case Outputs -> + { + return createLineageColumn(this, alias, false, false); + } + case Properties -> + { + return createPropertiesColumn(alias); + } + case SampleState -> + { + boolean statusEnabled = SampleStatusService.get().supportsSampleStatus() && !SampleStatusService.get().getAllProjectStates(getContainer()).isEmpty(); + var ret = wrapColumn(alias, _rootTable.getColumn(column.name())); + ret.setLabel("Status"); + ret.setHidden(!statusEnabled); + ret.setShownInDetailsView(statusEnabled); + ret.setShownInInsertView(statusEnabled); + ret.setShownInUpdateView(statusEnabled); + ret.setRemapMissingBehavior(SimpleTranslator.RemapMissingBehavior.Error); + ret.setFk(new QueryForeignKey.Builder(getUserSchema(), getSampleStatusLookupContainerFilter()) + .schema(getExpSchema()).table(ExpSchema.TableType.SampleStatus).display("Label")); + return ret; + } + case AliquotCount -> + { + var ret = wrapColumn(alias, _rootTable.getColumn(AliquotCount.name())); + ret.setLabel(ALIQUOT_COUNT_LABEL); + return ret; + } + case AliquotVolume -> + { + var ret = wrapColumn(alias, _rootTable.getColumn(AliquotVolume.name())); + ret.setLabel(ALIQUOT_VOLUME_LABEL); + return ret; + } + case AvailableAliquotVolume -> + { + var ret = wrapColumn(alias, _rootTable.getColumn(AvailableAliquotVolume.name())); + ret.setLabel(AVAILABLE_ALIQUOT_VOLUME_LABEL); + return ret; + } + case AvailableAliquotCount -> + { + var ret = wrapColumn(alias, _rootTable.getColumn(AvailableAliquotCount.name())); + ret.setLabel(AVAILABLE_ALIQUOT_COUNT_LABEL); + return ret; + } + case AliquotUnit -> + { + var ret = wrapColumn(alias, _rootTable.getColumn("AliquotUnit")); + ret.setShownInDetailsView(false); + return ret; + } + case MaterialExpDate -> + { + var ret = wrapColumn(alias, _rootTable.getColumn("MaterialExpDate")); + ret.setLabel("Expiration Date"); + ret.setShownInDetailsView(true); + ret.setShownInInsertView(true); + ret.setShownInUpdateView(true); + return ret; + } + default -> throw new IllegalArgumentException("Unknown column " + column); + } + } + + @Override + public MutableColumnInfo createPropertyColumn(String alias) + { + var ret = super.createPropertyColumn(alias); + if (_ss != null) + { + final TableInfo t = _ss.getTinfo(); + if (t != null) + { + ret.setFk(new LookupForeignKey() + { + @Override + public TableInfo getLookupTableInfo() + { + return t; + } + + @Override + protected ColumnInfo getPkColumn(TableInfo table) + { + return t.getColumn("lsid"); + } + }); + } + } + ret.setIsUnselectable(true); + ret.setDescription("A holder for any custom fields associated with this sample"); + ret.setHidden(true); + return ret; + } + + private Unit getSampleTypeUnit() + { + Unit typeUnit = null; + if (_ss != null && _ss.getMetricUnit() != null) + typeUnit = Unit.fromName(_ss.getMetricUnit()); + return typeUnit; + } + + private void setSampleType(@Nullable ExpSampleType st) + { + checkLocked(); + if (_ss != null) + { + throw new IllegalStateException("Cannot unset sample type"); + } + if (st != null && !(st instanceof ExpSampleTypeImpl)) + { + throw new IllegalArgumentException("Expected sample type to be an instance of " + ExpSampleTypeImpl.class.getName() + " but was a " + st.getClass().getName()); + } + _ss = (ExpSampleTypeImpl) st; + if (_ss != null) + { + setPublicSchemaName(SamplesSchema.SCHEMA_NAME); + setName(st.getName()); + + String description = _ss.getDescription(); + if (StringUtils.isEmpty(description)) + description = "Contains one row per sample in the " + _ss.getName() + " sample type"; + setDescription(description); + + if (canUserAccessPhi()) + { + ActionURL url = PageFlowUtil.urlProvider(ExperimentUrls.class).getImportSamplesURL(getContainer(), _ss.getName()); + setImportURL(new DetailsURL(url)); + } + } + } + + public ExpSampleType getSampleType() + { + return _ss; + } + + @Override + protected void populateColumns() + { + var st = getSampleType(); + var rowIdCol = addColumn(Column.RowId); + addColumn(Column.MaterialSourceId); + addColumn(Column.SourceProtocolApplication); + addColumn(Column.SourceApplicationInput); + addColumn(Column.RunApplication); + addColumn(Column.RunApplicationOutput); + addColumn(Column.SourceProtocolLSID); + + var nameCol = addColumn(Column.Name); + if (st != null && st.hasNameAsIdCol()) + { + // Show the Name field but don't mark is as required when using name expressions + if (st.hasNameExpression()) + { + var nameExpression = st.getNameExpression(); + nameCol.setNameExpression(nameExpression); + nameCol.setNullable(true); + String nameExpressionPreview = getExpNameExpressionPreview(getUserSchema().getSchemaName(), st.getName(), getUserSchema().getUser()); + String desc = appendNameExpressionDescription(nameCol.getDescription(), nameExpression, nameExpressionPreview); + nameCol.setDescription(desc); + } + else + { + nameCol.setNullable(false); + } + } + else + { + nameCol.setReadOnly(true); + nameCol.setShownInInsertView(false); + } + + addColumn(Column.Alias); + addColumn(Column.Description); + + var typeColumnInfo = addColumn(Column.SampleSet); + typeColumnInfo.setFk(new QueryForeignKey(_userSchema, getContainerFilter(), ExpSchema.SCHEMA_NAME, getContainer(), null, ExpSchema.TableType.SampleSets.name(), "lsid", null) + { + @Override + protected ContainerFilter getLookupContainerFilter() + { + // Be sure that we can resolve the sample type if it's defined in a separate container. + // Same as CurrentPlusProjectAndShared but includes SampleSet's container as well. + // Issue 37982: Sample Type: Link to precursor sample type does not resolve correctly if sample has + // parents in current sample type and a sample type in the parent container + Set containers = new HashSet<>(); + if (null != st) + containers.add(st.getContainer()); + containers.add(getContainer()); + if (getContainer().getProject() != null) + containers.add(getContainer().getProject()); + containers.add(ContainerManager.getSharedContainer()); + ContainerFilter cf = new ContainerFilter.CurrentPlusExtras(_userSchema.getContainer(), _userSchema.getUser(), containers); + + if (null != _containerFilter && _containerFilter.getType() != ContainerFilter.Type.Current) + cf = new UnionContainerFilter(_containerFilter, cf); + return cf; + } + }); + + typeColumnInfo.setReadOnly(true); + typeColumnInfo.setUserEditable(false); + typeColumnInfo.setShownInInsertView(false); + + addColumn(Column.MaterialExpDate); + addContainerColumn(Column.Folder, null); + var runCol = addColumn(Column.Run); + runCol.setFk(new ExpSchema(_userSchema.getUser(), getContainer()).getRunIdForeignKey(getContainerFilter())); + runCol.setShownInInsertView(false); + runCol.setShownInUpdateView(false); + + var colLSID = addColumn(Column.LSID); + colLSID.setHidden(true); + colLSID.setReadOnly(true); + colLSID.setUserEditable(false); + colLSID.setShownInInsertView(false); + colLSID.setShownInDetailsView(false); + colLSID.setShownInUpdateView(false); + + var rootRowId = addColumn(Column.RootMaterialRowId); + rootRowId.setHidden(true); + rootRowId.setReadOnly(true); + rootRowId.setUserEditable(false); + rootRowId.setShownInInsertView(false); + rootRowId.setShownInDetailsView(false); + rootRowId.setShownInUpdateView(false); + + var aliquotParentLSID = addColumn(Column.AliquotedFromLSID); + aliquotParentLSID.setHidden(true); + aliquotParentLSID.setReadOnly(true); + aliquotParentLSID.setUserEditable(false); + aliquotParentLSID.setShownInInsertView(false); + aliquotParentLSID.setShownInDetailsView(false); + aliquotParentLSID.setShownInUpdateView(false); + + addColumn(Column.IsAliquot); + addColumn(Column.Created); + addColumn(Column.CreatedBy); + addColumn(Column.Modified); + addColumn(Column.ModifiedBy); + + List defaultCols = new ArrayList<>(); + defaultCols.add(FieldKey.fromParts(Column.Name)); + defaultCols.add(FieldKey.fromParts(Column.MaterialExpDate)); + boolean hasProductFolders = getContainer().hasProductFolders(); + if (hasProductFolders) + defaultCols.add(FieldKey.fromParts(Column.Folder)); + defaultCols.add(FieldKey.fromParts(Column.Run)); + + if (st == null) + defaultCols.add(FieldKey.fromParts(Column.SampleSet)); + + addColumn(Column.Flag); + + var statusColInfo = addColumn(Column.SampleState); + boolean statusEnabled = SampleStatusService.get().supportsSampleStatus() && !SampleStatusService.get().getAllProjectStates(getContainer()).isEmpty(); + statusColInfo.setShownInDetailsView(statusEnabled); + statusColInfo.setShownInInsertView(statusEnabled); + statusColInfo.setShownInUpdateView(statusEnabled); + statusColInfo.setHidden(!statusEnabled); + statusColInfo.setRemapMissingBehavior(SimpleTranslator.RemapMissingBehavior.Error); + if (statusEnabled) + defaultCols.add(FieldKey.fromParts(Column.SampleState)); + statusColInfo.setFk(new QueryForeignKey.Builder(getUserSchema(), getSampleStatusLookupContainerFilter()) + .schema(getExpSchema()).table(ExpSchema.TableType.SampleStatus).display("Label")); + + // TODO is this a real Domain??? + if (st != null && !"urn:lsid:labkey.com:SampleSource:Default".equals(st.getDomain().getTypeURI())) + { + defaultCols.add(FieldKey.fromParts(Column.Flag)); + addSampleTypeColumns(st, defaultCols); + + setName(_ss.getName()); + + ActionURL gridUrl = new ActionURL(ExperimentController.ShowSampleTypeAction.class, getContainer()); + gridUrl.addParameter("rowId", st.getRowId()); + setGridURL(new DetailsURL(gridUrl)); + } + + List calculatedFieldKeys = DomainUtil.getCalculatedFieldsForDefaultView(this); + defaultCols.addAll(calculatedFieldKeys); + + addColumn(Column.AliquotCount); + addColumn(Column.AliquotVolume); + addColumn(Column.AliquotUnit); + addColumn(Column.AvailableAliquotCount); + addColumn(Column.AvailableAliquotVolume); + + addColumn(Column.StoredAmount); + defaultCols.add(FieldKey.fromParts(Column.StoredAmount)); + + addColumn(Column.Units); + defaultCols.add(FieldKey.fromParts(Column.Units)); + + var rawAmountColumn = addColumn(Column.RawAmount); + rawAmountColumn.setDisplayColumnFactory(new DisplayColumnFactory() + { + @Override + public DisplayColumn createRenderer(ColumnInfo colInfo) + { + return new DataColumn(colInfo) + { + @Override + public void addQueryFieldKeys(Set keys) + { + super.addQueryFieldKeys(keys); + keys.add(FieldKey.fromParts(Column.StoredAmount)); + + } + }; + } + }); + rawAmountColumn.setHidden(true); + rawAmountColumn.setShownInDetailsView(false); + rawAmountColumn.setShownInInsertView(false); + rawAmountColumn.setShownInUpdateView(false); + + var rawUnitsColumn = addColumn(Column.RawUnits); + rawUnitsColumn.setDisplayColumnFactory(new DisplayColumnFactory() + { + @Override + public DisplayColumn createRenderer(ColumnInfo colInfo) + { + return new DataColumn(colInfo) + { + @Override + public void addQueryFieldKeys(Set keys) + { + super.addQueryFieldKeys(keys); + keys.add(FieldKey.fromParts(Column.Units)); + + } + }; + } + }); + rawUnitsColumn.setHidden(true); + rawUnitsColumn.setShownInDetailsView(false); + rawUnitsColumn.setShownInInsertView(false); + rawUnitsColumn.setShownInUpdateView(false); + + if (InventoryService.get() != null && (st == null || !st.isMedia())) + defaultCols.addAll(InventoryService.get().addInventoryStatusColumns(st == null ? null : st.getMetricUnit(), this, getContainer(), _userSchema.getUser())); + + SQLFragment sql; + UserSchema plateUserSchema; + // Issue 53194 : this would be the case for linked to study samples. The contextual role is set up from the study dataset + // for the source sample, we want to allow the plate schema to inherit any contextual roles to allow querying + // against tables in that schema. + if (_userSchema instanceof UserSchema.HasContextualRoles samplesSchema && !samplesSchema.getContextualRoles().isEmpty()) + plateUserSchema = AssayPlateMetadataService.get().getPlateSchema(_userSchema, samplesSchema.getContextualRoles()); + else + plateUserSchema = QueryService.get().getUserSchema(_userSchema.getUser(), _userSchema.getContainer(), "plate"); + + if (plateUserSchema != null && plateUserSchema.getTable("Well") != null) + { + String rowIdField = ExprColumn.STR_TABLE_ALIAS + "." + Column.RowId.name(); + SQLFragment existsSubquery = new SQLFragment() + .append("SELECT 1 FROM ") + .append(plateUserSchema.getTable("Well"), "well") + .append(" WHERE well.sampleid = ").append(rowIdField); + + sql = new SQLFragment() + .append("CASE WHEN EXISTS (") + .append(existsSubquery) + .append(") THEN 'Plated' ") + .append("WHEN ").append(ExprColumn.STR_TABLE_ALIAS).append(".RowId").append(" IS NOT NULL THEN 'Not Plated' ")// Issue 52745 + .append("ELSE NULL END"); + } + else + { + sql = new SQLFragment("(SELECT NULL)"); + } + var col = new ExprColumn(this, Column.IsPlated.name(), sql, JdbcType.VARCHAR); + col.setDescription("Whether the sample that has been plated, if plating is supported."); + col.setUserEditable(false); + col.setReadOnly(true); + col.setShownInDetailsView(false); + col.setShownInInsertView(false); + col.setShownInUpdateView(false); + if (plateUserSchema != null) + col.setURL(DetailsURL.fromString("plate-isPlated.api?sampleId=${" + Column.RowId.name() + "}")); + addColumn(col); + + addVocabularyDomains(); + + addColumn(Column.Properties); + + var colInputs = addColumn(Column.Inputs); + addMethod("Inputs", new LineageMethod(colInputs, true), Set.of(colInputs.getFieldKey())); + + var colOutputs = addColumn(Column.Outputs); + addMethod("Outputs", new LineageMethod(colOutputs, false), Set.of(colOutputs.getFieldKey())); + + addExpObjectMethod(); + + ActionURL detailsUrl = new ActionURL(ExperimentController.ShowMaterialAction.class, getContainer()); + DetailsURL url = new DetailsURL(detailsUrl, Collections.singletonMap("rowId", "RowId"), NullResult); + nameCol.setURL(url); + rowIdCol.setURL(url); + setDetailsURL(url); + + if (canUserAccessPhi()) + { + ActionURL updateActionURL = PageFlowUtil.urlProvider(ExperimentUrls.class).getUpdateMaterialQueryRowAction(getContainer(), this); + setUpdateURL(new DetailsURL(updateActionURL, Collections.singletonMap("RowId", "RowId"))); + + ActionURL insertActionURL = PageFlowUtil.urlProvider(ExperimentUrls.class).getInsertMaterialQueryRowAction(getContainer(), this); + setInsertURL(new DetailsURL(insertActionURL)); + } + else + { + setImportURL(LINK_DISABLER); + setInsertURL(LINK_DISABLER); + setUpdateURL(LINK_DISABLER); + } + + setTitleColumn(Column.Name.toString()); + + setDefaultVisibleColumns(defaultCols); + + MutableColumnInfo lineageLookup = ClosureQueryHelper.createAncestorLookupColumnInfo("Ancestors", this, _rootTable.getColumn("rowid"), _ss, true); + addColumn(lineageLookup); + } + + private ContainerFilter getSampleStatusLookupContainerFilter() + { + // The default lookup container filter is Current, but we want to have the default be CurrentPlusProjectAndShared + // for the sample status lookup since in the app project context we want to share status definitions across + // a given project instead of creating duplicate statuses in each subfolder project. + ContainerFilter.Type type = QueryService.get().getContainerFilterTypeForLookups(getContainer()); + type = type == null ? ContainerFilter.Type.CurrentPlusProjectAndShared : type; + return type.create(getUserSchema()); + } + + @Override + public Domain getDomain() + { + return getDomain(false); + } + + @Override + public Domain getDomain(boolean forUpdate) + { + return _ss == null ? null : _ss.getDomain(forUpdate); + } + + + public static String appendNameExpressionDescription(String currentDescription, String nameExpression, String nameExpressionPreview) + { + if (nameExpression == null) + return currentDescription; + + StringBuilder sb = new StringBuilder(); + if (currentDescription != null && !currentDescription.isEmpty()) + { + sb.append(currentDescription); + if (!currentDescription.endsWith(".")) + sb.append("."); + sb.append("\n"); + } + + sb.append("\nIf not provided, a unique name will be generated from the expression:\n"); + sb.append(nameExpression); + sb.append("."); + if (!StringUtils.isEmpty(nameExpressionPreview)) + { + sb.append("\nExample of name that will be generated from the current pattern: \n"); + sb.append(nameExpressionPreview); + } + + return sb.toString(); + } + + private void addSampleTypeColumns(ExpSampleType st, List visibleColumns) + { + TableInfo dbTable = ((ExpSampleTypeImpl)st).getTinfo(); + if (null == dbTable) + return; + + UserSchema schema = getUserSchema(); + Domain domain = st.getDomain(); + ColumnInfo rowIdColumn = getColumn(Column.RowId); + ColumnInfo lsidColumn = getColumn(Column.LSID); + ColumnInfo nameColumn = getColumn(Column.Name); + + visibleColumns.remove(FieldKey.fromParts(Column.Run.name())); + + // When not using name expressions, mark the ID columns as required. + // NOTE: If not explicitly set, the first domain property will be chosen as the ID column. + final List idCols = st.hasNameExpression() ? Collections.emptyList() : st.getIdCols(); + + Set mvColumns = domain.getProperties().stream() + .filter(ImportAliasable::isMvEnabled) + .map(dp -> FieldKey.fromParts(dp.getPropertyDescriptor().getMvIndicatorStorageColumnName())) + .collect(Collectors.toSet()); + + for (ColumnInfo dbColumn : dbTable.getColumns()) + { + // Don't include PHI columns in full text search index + // CONSIDER: Can we move this to a base class? Maybe in .addColumn() + if (schema.getUser().isSearchUser() && !dbColumn.getPHI().isLevelAllowed(PHI.NotPHI)) + continue; + + if ( + rowIdColumn.getFieldKey().equals(dbColumn.getFieldKey()) || + lsidColumn.getFieldKey().equals(dbColumn.getFieldKey()) || + nameColumn.getFieldKey().equals(dbColumn.getFieldKey()) + ) + { + continue; + } + + var wrapped = wrapColumnFromJoinedTable(dbColumn.getName(), dbColumn); + + // TODO missing values? comments? flags? + DomainProperty dp = domain.getPropertyByURI(dbColumn.getPropertyURI()); + var propColumn = copyColumnFromJoinedTable(null==dp ? dbColumn.getName() : dp.getName(), wrapped); + if (propColumn.getName().equalsIgnoreCase("genid")) + { + propColumn.setHidden(true); + propColumn.setUserEditable(false); + propColumn.setShownInDetailsView(false); + propColumn.setShownInInsertView(false); + propColumn.setShownInUpdateView(false); + } + if (null != dp) + { + PropertyColumn.copyAttributes(schema.getUser(), propColumn, dp.getPropertyDescriptor(), schema.getContainer(), + SchemaKey.fromParts("samples"), st.getName(), FieldKey.fromParts("RowId"), null, getLookupContainerFilter()); + + if (idCols.contains(dp)) + { + propColumn.setNullable(false); + propColumn.setDisplayColumnFactory(new IdColumnRendererFactory()); + } + + // Issue 38341: domain designer advanced settings 'show in default view' setting is not respected + if (!propColumn.isHidden()) + { + visibleColumns.add(propColumn.getFieldKey()); + } + + if (propColumn.isMvEnabled()) + { + // The column in the physical table has a "_MVIndicator" suffix, but we want to expose + // it with a "MVIndicator" suffix (no underscore) + var mvColumn = new AliasedColumn(this, dp.getName() + MvColumn.MV_INDICATOR_SUFFIX, + StorageProvisioner.get().getMvIndicatorColumn(dbTable, dp.getPropertyDescriptor(), "No MV column found for '" + dp.getName() + "' in sample type '" + getName() + "'")); + mvColumn.setLabel(dp.getLabel() != null ? dp.getLabel() : dp.getName() + " MV Indicator"); + mvColumn.setSqlTypeName("VARCHAR"); + mvColumn.setPropertyURI(dp.getPropertyURI()); + mvColumn.setNullable(true); + mvColumn.setUserEditable(false); + mvColumn.setHidden(true); + mvColumn.setMvIndicatorColumn(true); + + addColumn(mvColumn); + propColumn.setMvColumnName(FieldKey.fromParts(dp.getName() + MvColumn.MV_INDICATOR_SUFFIX)); + } + } + + if (!mvColumns.contains(propColumn.getFieldKey())) + addColumn(propColumn); + + } + + setDefaultVisibleColumns(visibleColumns); + } + + // These are mostly fields that are wrapped by fields with different names (see createColumn()) + // we could handle each case separately, but this is easier + static final Set wrappedFieldKeys = Set.of( + new FieldKey(null, "objectid"), + new FieldKey(null, "RowId"), + new FieldKey(null, "LSID"), // Flag + new FieldKey(null, "SourceApplicationId"), // SourceProtocolApplication + new FieldKey(null, "runId"), // Run, RunApplication + new FieldKey(null, "CpasType")); // SampleSet + static final Set ALL_COLUMNS = Set.of(); + + private @NotNull Set computeInnerSelectedColumns(Set selectedColumns) + { + if (null == selectedColumns) + return ALL_COLUMNS; + selectedColumns = new TreeSet<>(selectedColumns); + if (selectedColumns.contains(new FieldKey(null, StoredAmount))) + selectedColumns.add(new FieldKey(null, Units)); + if (selectedColumns.contains(new FieldKey(null, ExpMaterial.ALIQUOTED_FROM_INPUT))) + selectedColumns.add(new FieldKey(null, Column.AliquotedFromLSID.name())); + if (selectedColumns.contains(new FieldKey(null, Column.IsAliquot.name()))) + selectedColumns.add(new FieldKey(null, Column.RootMaterialRowId.name())); + selectedColumns.addAll(wrappedFieldKeys); + if (null != getFilter()) + selectedColumns.addAll(getFilter().getAllFieldKeys()); + return selectedColumns; + } + + @NotNull + @Override + public SQLFragment getFromSQL(String alias) + { + return getFromSQL(alias, null); + } + + @Override + public SQLFragment getFromSQLExpanded(String alias, Set selectedColumns) + { + SQLFragment sql = new SQLFragment("("); + boolean usedMaterialized; + + + // SELECT FROM + /* NOTE We want to avoid caching in paths where the table is actively being updated (e.g. loadRows) + * Unfortunately, we don't _really_ know when this is, but if we in a transaction that's a good guess. + * Also, we may use RemapCache for material lookup outside a transaction + */ + boolean onlyMaterialColums = false; + if (null != selectedColumns && !selectedColumns.isEmpty()) + onlyMaterialColums = selectedColumns.stream().allMatch(fk -> fk.getName().equalsIgnoreCase("Folder") || null != _rootTable.getColumn(fk)); + if (!onlyMaterialColums && null != _ss && null != _ss.getTinfo() && !getExpSchema().getDbSchema().getScope().isTransactionActive()) + { + sql.append(getMaterializedSQL()); + usedMaterialized = true; + } + else + { + sql.append(getJoinSQL(selectedColumns)); + usedMaterialized = false; + } + + // WHERE + SQLFragment filterFrag = getFilter().getSQLFragment(_rootTable, null); + sql.append("\n").append(filterFrag); + if (_ss != null && !usedMaterialized) + { + if (!filterFrag.isEmpty()) + sql.append(" AND "); + else + sql.append(" WHERE "); + sql.append("CpasType = ").appendValue(_ss.getLSID()); + } + sql.append(") ").appendIdentifier(alias); + + return getTransformedFromSQL(sql); + } + + @Override + public void setSupportTableRules(boolean b) + { + this._supportTableRules = b; + } + + @Override + public boolean supportTableRules() // intentional override + { + return _supportTableRules; + } + + @Override + protected @NotNull TableRules findTableRules() + { + Container definitionContainer = getUserSchema().getContainer(); + if (null != _ss) + definitionContainer = _ss.getContainer(); + return TableRulesManager.get().getTableRules(definitionContainer, getUserSchema().getUser(), getUserSchema().getContainer()); + } + + + static class InvalidationCounters + { + public final AtomicLong update, insert, delete, rollup; + InvalidationCounters() + { + long l = System.currentTimeMillis(); + update = new AtomicLong(l); + insert = new AtomicLong(l); + delete = new AtomicLong(l); + rollup = new AtomicLong(l); + } + } + + static final BlockingCache _materializedQueries = CacheManager.getBlockingStringKeyCache(CacheManager.UNLIMITED, CacheManager.HOUR, "materialized sample types", null); + static final Map _invalidationCounters = Collections.synchronizedMap(new HashMap<>()); + static final AtomicBoolean initializedListeners = new AtomicBoolean(false); + + // used by SampleTypeServiceImpl.refreshSampleTypeMaterializedView() + public static void refreshMaterializedView(final String lsid, SampleTypeServiceImpl.SampleChangeType reason) + { + var scope = ExperimentServiceImpl.getExpSchema().getScope(); + var runnable = new RefreshMaterializedViewRunnable(lsid, reason); + scope.addCommitTask(runnable, DbScope.CommitTaskOption.POSTCOMMIT); + } + + private static class RefreshMaterializedViewRunnable implements Runnable + { + private final String _lsid; + private final SampleTypeServiceImpl.SampleChangeType _reason; + + public RefreshMaterializedViewRunnable(String lsid, SampleTypeServiceImpl.SampleChangeType reason) + { + _lsid = lsid; + _reason = reason; + } + + @Override + public void run() + { + if (_reason == schema) + { + /* NOTE: MaterializedQueryHelper can detect data changes and refresh the materialized view using the provided SQL. + * It does not handle schema changes where the SQL itself needs to be updated. In this case, we remove the + * MQH from the cache to force the SQL to be regenerated. + */ + _materializedQueries.remove(_lsid); + return; + } + var counters = getInvalidateCounters(_lsid); + switch (_reason) + { + case insert -> counters.insert.incrementAndGet(); + case rollup -> counters.rollup.incrementAndGet(); + case update -> counters.update.incrementAndGet(); + case delete -> counters.delete.incrementAndGet(); + default -> throw new IllegalStateException("Unexpected value: " + _reason); + } + } + + @Override + public boolean equals(Object obj) + { + return obj instanceof RefreshMaterializedViewRunnable other && _lsid.equals(other._lsid) && _reason.equals(other._reason); + } + } + + private static InvalidationCounters getInvalidateCounters(String lsid) + { + if (!initializedListeners.getAndSet(true)) + { + CacheManager.addListener(_invalidationCounters::clear); + } + return _invalidationCounters.computeIfAbsent(lsid, (unused) -> + new InvalidationCounters() + ); + } + + /* SELECT and JOIN, does not include WHERE, same as getJoinSQL() */ + private SQLFragment getMaterializedSQL() + { + if (null == _ss) + return getJoinSQL(null); + + var mqh = _materializedQueries.get(_ss.getLSID(), null, (unusedKey, unusedArg) -> + { + /* NOTE: MaterializedQueryHelper does have a pattern to help with detecting schema changes. + * Previously it has been used on non-provisioned tables. It might be helpful to have a pattern, + * even if just to help with race-conditions. + * + * Maybe have a callback to generate the SQL dynamically, and verify that the sql is unchanged. + */ + SQLFragment viewSql = getJoinSQL(null).append(" WHERE CpasType = ").appendValue(_ss.getLSID()); + return (_MaterializedQueryHelper) new _MaterializedQueryHelper.Builder(_ss.getLSID(), "", getExpSchema().getDbSchema().getScope(), viewSql) + .addIndex("CREATE UNIQUE INDEX uq_${NAME}_rowid ON temp.${NAME} (rowid)") + .addIndex("CREATE UNIQUE INDEX uq_${NAME}_lsid ON temp.${NAME} (lsid)") + .addIndex("CREATE INDEX idx_${NAME}_container ON temp.${NAME} (container)") + .addIndex("CREATE INDEX idx_${NAME}_root ON temp.${NAME} (rootmaterialrowid)") + .addInvalidCheck(() -> String.valueOf(getInvalidateCounters(_ss.getLSID()).update.get())) + .build(); + }); + return new SQLFragment("SELECT * FROM ").append(mqh.getFromSql("_cached_view_")); + } + + + /** + * MaterializedQueryHelper has a built-in mechanism for tracking when a temp table needs to be recomputed. + * It does not help with incremental updates (except for providing the upsert() method). + * _MaterializedQueryHelper and _Materialized copy the pattern using class Invalidator. + */ + static class _MaterializedQueryHelper extends MaterializedQueryHelper + { + final String _lsid; + + static class Builder extends MaterializedQueryHelper.Builder + { + String _lsid; + + public Builder(String lsid, String prefix, DbScope scope, SQLFragment select) + { + super(prefix, scope, select); + this._lsid = lsid; + } + + @Override + public _MaterializedQueryHelper build() + { + return new _MaterializedQueryHelper(_lsid, _prefix, _scope, _select, _uptodate, _supplier, _indexes, _max, _isSelectInto); + } + } + + _MaterializedQueryHelper(String lsid, String prefix, DbScope scope, SQLFragment select, @Nullable SQLFragment uptodate, Supplier supplier, @Nullable Collection indexes, long maxTimeToCache, + boolean isSelectIntoSql) + { + super(prefix, scope, select, uptodate, supplier, indexes, maxTimeToCache, isSelectIntoSql); + this._lsid = lsid; + } + + @Override + protected Materialized createMaterialized(String txCacheKey) + { + DbSchema temp = DbSchema.getTemp(); + String name = _prefix + "_" + GUID.makeHash(); + _Materialized materialized = new _Materialized(this, name, txCacheKey, HeartBeat.currentTimeMillis(), "\"" + temp.getName() + "\".\"" + name + "\""); + initMaterialized(materialized); + return materialized; + } + + @Override + protected void incrementalUpdateBeforeSelect(Materialized m) + { + _Materialized materialized = (_Materialized) m; + + boolean lockAcquired = false; + try + { + lockAcquired = materialized.getLock().tryLock(1, TimeUnit.MINUTES); + if (Materialized.LoadingState.ERROR == materialized._loadingState.get()) + throw materialized._loadException; + + if (!materialized.incrementalDeleteCheck.stillValid(0)) + executeIncrementalDelete(); + if (!materialized.incrementalRollupCheck.stillValid(0)) + executeIncrementalRollup(); + if (!materialized.incrementalInsertCheck.stillValid(0)) + executeIncrementalInsert(); + } + catch (RuntimeException|InterruptedException ex) + { + RuntimeException rex = UnexpectedException.wrap(ex); + materialized.setError(rex); + // The only time I'd expect an error is due to a schema change race-condition, but that can happen in any code path. + + // Ensure that next refresh starts clean + _materializedQueries.remove(_lsid); + getInvalidateCounters(_lsid).update.incrementAndGet(); + throw rex; + } + finally + { + if (lockAcquired) + materialized.getLock().unlock(); + } + } + + void upsertWithRetry(SQLFragment sql) + { + // not actually read-only, but we don't want to start an explicit transaction + _scope.executeWithRetryReadOnly((tx) -> upsert(sql)); + } + + void executeIncrementalInsert() + { + SQLFragment incremental = new SQLFragment("INSERT INTO temp.${NAME}\n") + .append("SELECT * FROM (") + .append(getViewSourceSql()).append(") viewsource_\n") + .append("WHERE rowid > (SELECT COALESCE(MAX(rowid),0) FROM temp.${NAME})"); + upsertWithRetry(incremental); + } + + void executeIncrementalDelete() + { + var d = CoreSchema.getInstance().getSchema().getSqlDialect(); + // POSTGRES bug??? the obvious query is _very_ slow O(n^2) + // DELETE FROM temp.${NAME} WHERE rowid NOT IN (SELECT rowid FROM exp.material WHERE cpastype = <<_lsid>>) + SQLFragment incremental = new SQLFragment() + .append("WITH deleted AS (SELECT rowid FROM temp.${NAME} EXCEPT SELECT rowid FROM exp.material WHERE cpastype = ").appendValue(_lsid,d).append(")\n") + .append("DELETE FROM temp.${NAME} WHERE rowid IN (SELECT rowid from deleted)\n"); + upsertWithRetry(incremental); + } + + void executeIncrementalRollup() + { + var d = CoreSchema.getInstance().getSchema().getSqlDialect(); + SQLFragment incremental = new SQLFragment(); + if (d.isPostgreSQL()) + { + incremental + .append("UPDATE temp.${NAME} AS st\n") + .append("SET aliquotcount = expm.aliquotcount, availablealiquotcount = expm.availablealiquotcount, aliquotvolume = expm.aliquotvolume, availablealiquotvolume = expm.availablealiquotvolume, aliquotunit = expm.aliquotunit\n") + .append("FROM exp.Material AS expm\n") + .append("WHERE expm.rowid = st.rowid AND expm.cpastype = ").appendValue(_lsid,d).append(" AND (\n") + .append(" st.aliquotcount IS DISTINCT FROM expm.aliquotcount OR ") + .append(" st.availablealiquotcount IS DISTINCT FROM expm.availablealiquotcount OR ") + .append(" st.aliquotvolume IS DISTINCT FROM expm.aliquotvolume OR ") + .append(" st.availablealiquotvolume IS DISTINCT FROM expm.availablealiquotvolume OR ") + .append(" st.aliquotunit IS DISTINCT FROM expm.aliquotunit") + .append(")"); + } + else + { + // SQL Server 2022 supports IS DISTINCT FROM + incremental + .append("UPDATE st\n") + .append("SET aliquotcount = expm.aliquotcount, availablealiquotcount = expm.availablealiquotcount, aliquotvolume = expm.aliquotvolume, availablealiquotvolume = expm.availablealiquotvolume, aliquotunit = expm.aliquotunit\n") + .append("FROM temp.${NAME} st, exp.Material expm\n") + .append("WHERE expm.rowid = st.rowid AND expm.cpastype = ").appendValue(_lsid,d).append(" AND (\n") + .append(" COALESCE(st.aliquotcount,-2147483648) <> COALESCE(expm.aliquotcount,-2147483648) OR ") + .append(" COALESCE(st.availablealiquotcount,-2147483648) <> COALESCE(expm.availablealiquotcount,-2147483648) OR ") + .append(" COALESCE(st.aliquotvolume,-2147483648) <> COALESCE(expm.aliquotvolume,-2147483648) OR ") + .append(" COALESCE(st.availablealiquotvolume,-2147483648) <> COALESCE(expm.availablealiquotvolume,-2147483648) OR ") + .append(" COALESCE(st.aliquotunit,'-') <> COALESCE(expm.aliquotunit,'-')") + .append(")"); + } + upsertWithRetry(incremental); + } + } + + static class _Materialized extends MaterializedQueryHelper.Materialized + { + final MaterializedQueryHelper.Invalidator incrementalInsertCheck; + final MaterializedQueryHelper.Invalidator incrementalRollupCheck; + final MaterializedQueryHelper.Invalidator incrementalDeleteCheck; + + _Materialized(_MaterializedQueryHelper mqh, String tableName, String cacheKey, long created, String sql) + { + super(mqh, tableName, cacheKey, created, sql); + final InvalidationCounters counters = getInvalidateCounters(mqh._lsid); + incrementalInsertCheck = new MaterializedQueryHelper.SupplierInvalidator(() -> String.valueOf(counters.insert.get())); + incrementalRollupCheck = new MaterializedQueryHelper.SupplierInvalidator(() -> String.valueOf(counters.rollup.get())); + incrementalDeleteCheck = new MaterializedQueryHelper.SupplierInvalidator(() -> String.valueOf(counters.delete.get())); + } + + @Override + public void reset() + { + super.reset(); + long now = HeartBeat.currentTimeMillis(); + incrementalInsertCheck.stillValid(now); + incrementalRollupCheck.stillValid(now); + incrementalDeleteCheck.stillValid(now); + } + + Lock getLock() + { + return _loadingLock; + } + } + + + /* SELECT and JOIN, does not include WHERE */ + private SQLFragment getJoinSQL(Set selectedColumns) + { + TableInfo provisioned = null == _ss ? null : _ss.getTinfo(); + Set provisionedCols = new CaseInsensitiveHashSet(provisioned != null ? provisioned.getColumnNameSet() : Collections.emptySet()); + provisionedCols.remove(Column.RowId.name()); + provisionedCols.remove(Column.LSID.name()); + provisionedCols.remove(Column.Name.name()); + boolean hasProvisionedColumns = containsProvisionedColumns(selectedColumns, provisionedCols); + + boolean hasSampleColumns = false; + boolean hasAliquotColumns = false; + + Set materialCols = new CaseInsensitiveHashSet(_rootTable.getColumnNameSet()); + selectedColumns = computeInnerSelectedColumns(selectedColumns); + + SQLFragment sql = new SQLFragment(); + sql.appendComment("", getSqlDialect()); + sql.append("SELECT "); + String comma = ""; + for (String materialCol : materialCols) + { + // don't need to generate SQL for columns that aren't selected + if (ALL_COLUMNS == selectedColumns || selectedColumns.contains(new FieldKey(null, materialCol))) + { + sql.append(comma).append("m.").appendIdentifier(materialCol); + comma = ", "; + } + } + if (null != provisioned && hasProvisionedColumns) + { + for (ColumnInfo propertyColumn : provisioned.getColumns()) + { + // don't select twice + if ( + Column.RowId.name().equalsIgnoreCase(propertyColumn.getColumnName()) || + Column.LSID.name().equalsIgnoreCase(propertyColumn.getColumnName()) || + Column.Name.name().equalsIgnoreCase(propertyColumn.getColumnName()) + ) + { + continue; + } + + // don't need to generate SQL for columns that aren't selected + if (ALL_COLUMNS == selectedColumns || selectedColumns.contains(propertyColumn.getFieldKey()) || propertyColumn.isMvIndicatorColumn()) + { + sql.append(comma); + boolean rootField = StringUtils.isEmpty(propertyColumn.getDerivationDataScope()) + || ExpSchema.DerivationDataScopeType.ParentOnly.name().equalsIgnoreCase(propertyColumn.getDerivationDataScope()); + if ("genid".equalsIgnoreCase(propertyColumn.getColumnName()) || propertyColumn.isUniqueIdField()) + { + sql.append(propertyColumn.getValueSql("m_aliquot")).append(" AS ").appendIdentifier(propertyColumn.getSelectIdentifier()); + hasAliquotColumns = true; + } + else if (rootField) + { + sql.append(propertyColumn.getValueSql("m_sample")).append(" AS ").appendIdentifier(propertyColumn.getSelectIdentifier()); + hasSampleColumns = true; + } + else + { + sql.append(propertyColumn.getValueSql("m_aliquot")).append(" AS ").appendIdentifier(propertyColumn.getSelectIdentifier()); + hasAliquotColumns = true; + } + comma = ", "; + } + } + } + + sql.append("\nFROM "); + sql.append(_rootTable, "m"); + if (hasSampleColumns) + sql.append(" INNER JOIN ").append(provisioned, "m_sample").append(" ON m.RootMaterialRowId = m_sample.RowId"); + if (hasAliquotColumns) + sql.append(" INNER JOIN ").append(provisioned, "m_aliquot").append(" ON m.RowId = m_aliquot.RowId"); + + sql.appendComment("", getSqlDialect()); + return sql; + } + + private class IdColumnRendererFactory implements DisplayColumnFactory + { + @Override + public DisplayColumn createRenderer(ColumnInfo colInfo) + { + return new IdColumnRenderer(colInfo); + } + } + + private static class IdColumnRenderer extends DataColumn + { + public IdColumnRenderer(ColumnInfo col) + { + super(col); + } + + @Override + protected boolean isDisabledInput(RenderContext ctx) + { + return !super.isDisabledInput() && ctx.getMode() != DataRegion.MODE_INSERT; + } + } + + private static class SampleTypeAmountDisplayColumn extends ExprColumn + { + public SampleTypeAmountDisplayColumn(TableInfo parent, String amountFieldName, String unitFieldName, String label, Set importAliases, Unit typeUnit) + { + super(parent, FieldKey.fromParts(amountFieldName), new SQLFragment( + "(CASE WHEN ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(unitFieldName) + .append(" = ? AND ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(amountFieldName) + .append(" IS NOT NULL THEN CAST(").append(ExprColumn.STR_TABLE_ALIAS + ".").append(amountFieldName) + .append(" / ? AS ") + .append(parent.getSqlDialect().isPostgreSQL() ? "DECIMAL" : "DOUBLE PRECISION") + .append(") ELSE ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(amountFieldName) + .append(" END)") + .add(typeUnit.getBase().toString()) + .add(typeUnit.getValue()), + JdbcType.DOUBLE); + + setLabel(label); + setImportAliasesSet(importAliases); + } + } + + private static class SampleTypeUnitDisplayColumn extends ExprColumn + { + public SampleTypeUnitDisplayColumn(TableInfo parent, String unitFieldName, Unit typeUnit) + { + super(parent, FieldKey.fromParts(Column.Units.name()), new SQLFragment( + "(CASE WHEN ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(unitFieldName) + .append(" = ? THEN ? ELSE ").append(ExprColumn.STR_TABLE_ALIAS + ".").append(unitFieldName) + .append(" END)") + .add(typeUnit.getBase().toString()) + .add(typeUnit.toString()), + JdbcType.VARCHAR); + } + } + + @Override + public QueryUpdateService getUpdateService() + { + return new SampleTypeUpdateServiceDI(this, _ss); + } + + @Override + public boolean hasPermission(@NotNull UserPrincipal user, @NotNull Class perm) + { + if (_ss == null) + { + // Allow read and delete for exp.Materials. + // Don't allow insert/update on exp.Materials without a sample type. + if (perm == DeletePermission.class || perm == ReadPermission.class) + return getContainer().hasPermission(user, perm); + return false; + } + + if (_ss.isMedia() && perm == ReadPermission.class) + return getContainer().hasPermission(user, MediaReadPermission.class); + + return super.hasPermission(user, perm); + } + + @NotNull + @Override + public List getUniqueIndices() + { + // Rewrite the "idx_material_ak" unique index over "Folder", "SampleSet", "Name" to just "Name" + // Issue 25397: Don't include the "idx_material_ak" index if the "Name" column hasn't been added to the table. + // Some FKs to ExpMaterialTable don't include the "Name" column (e.g. NabBaseTable.Specimen) + String indexName = "idx_material_ak"; + List ret = new ArrayList<>(super.getUniqueIndices()); + if (getColumn("Name") != null) + ret.add(new IndexDefinition(indexName, IndexType.Unique, Arrays.asList(getColumn("Name")), null)); + else + ret.removeIf( def -> def.name().equals(indexName)); + return Collections.unmodifiableList(ret); + } + + + // + // UpdatableTableInfo + // + + + @Override + public @Nullable Long getOwnerObjectId() + { + return OntologyManager.ensureObject(_ss.getContainer(), _ss.getLSID(), (Long) null); + } + + @Nullable + @Override + public CaseInsensitiveHashMap remapSchemaColumns() + { + CaseInsensitiveHashMap m = new CaseInsensitiveHashMap<>(); + + if (null != getRealTable().getColumn("container") && null != getColumn("folder")) + { + m.put("container", "folder"); + } + + for (ColumnInfo col : getColumns()) + { + if (col.getMvColumnName() != null) + m.put(col.getName() + "_" + MvColumn.MV_INDICATOR_SUFFIX, col.getMvColumnName().getName()); + } + + return m; + } + + @Override + public Set getAltMergeKeys(DataIteratorContext context) + { + if (context.getInsertOption().updateOnly && context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate)) + return getAltKeysForUpdate(); + + return MATERIAL_ALT_MERGE_KEYS; + } + + @NotNull + @Override + public Set getAltKeysForUpdate() + { + return MATERIAL_ALT_UPDATE_KEYS; + } + + @Override + @NotNull + public List> getAdditionalRequiredInsertColumns() + { + if (getSampleType() == null) + return Collections.emptyList(); + + try + { + return getRequiredParentImportFields(getSampleType().getRequiredImportAliases()); + } + catch (IOException e) + { + return Collections.emptyList(); + } + } + + @Override + public DataIteratorBuilder persistRows(DataIteratorBuilder data, DataIteratorContext context) + { + TableInfo propertiesTable = _ss.getTinfo(); + + // The specimens sample type doesn't have a properties table + if (propertiesTable == null) + { + return data; + } + + long sampleTypeObjectId = requireNonNull(getOwnerObjectId()); + + // TODO: subclass PersistDataIteratorBuilder to index Materials! not DataClass! + try + { + var persist = new ExpDataIterators.PersistDataIteratorBuilder(data, this, propertiesTable, _ss, getUserSchema().getContainer(), getUserSchema().getUser(), _ss.getImportAliases(), sampleTypeObjectId) + .setFileLinkDirectory(SAMPLETYPE_FILE_DIRECTORY); + ExperimentServiceImpl experimentServiceImpl = ExperimentServiceImpl.get(); + SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(getContainer(), SearchService.PRIORITY.modified); + + persist.setIndexFunction(searchIndexDataKeys -> propertiesTable.getSchema().getScope().addCommitTask(() -> + { + List lsids = searchIndexDataKeys.lsids(); + List orderedRowIds = searchIndexDataKeys.orderedRowIds(); + + // Issue 51263: order by RowId to reduce deadlock + ListUtils.partition(orderedRowIds, 100).forEach(sublist -> + queue.addRunnable((q) -> + { + for (ExpMaterialImpl expMaterial : experimentServiceImpl.getExpMaterials(sublist)) + expMaterial.index(q, this); + }) + ); + + ListUtils.partition(lsids, 100).forEach(sublist -> + queue.addRunnable((q) -> + { + for (ExpMaterialImpl expMaterial : experimentServiceImpl.getExpMaterialsByLsid(sublist)) + expMaterial.index(q, this); + }) + ); + }, DbScope.CommitTaskOption.POSTCOMMIT) + ); + + DataIteratorBuilder builder = LoggingDataIterator.wrap(persist); + return LoggingDataIterator.wrap(new AliasDataIteratorBuilder(builder, getUserSchema().getContainer(), getUserSchema().getUser(), ExperimentService.get().getTinfoMaterialAliasMap(), _ss, true)); + } + catch (IOException e) + { + throw new UncheckedIOException(e); + } + } + + @Override + @NotNull + public AuditBehaviorType getDefaultAuditBehavior() + { + return AuditBehaviorType.DETAILED; + } + + static final Set excludeFromDetailedAuditField; + static + { + var set = new CaseInsensitiveHashSet(); + set.addAll(TableInfo.defaultExcludedDetailedUpdateAuditFields); + set.addAll(ExpDataIterators.NOT_FOR_UPDATE); + // We don't want the inventory columns to show up in the sample timeline audit record; + // they are captured in their own audit record. + set.addAll(InventoryService.InventoryStatusColumn.names()); + excludeFromDetailedAuditField = Collections.unmodifiableSet(set); + } + + @Override + public @NotNull Set getExcludedDetailedUpdateAuditFields() + { + // uniqueId fields don't change in reality, so exclude them from the audit updates + Set excluded = new CaseInsensitiveHashSet(); + excluded.addAll(this.getUniqueIdFields()); + excluded.addAll(excludeFromDetailedAuditField); + return excluded; + } + + @Override + public List> getImportTemplates(ViewContext ctx) + { + // respect any metadata overrides + if (getRawImportTemplates() != null) + return super.getImportTemplates(ctx); + + List> templates = new ArrayList<>(); + ActionURL url = PageFlowUtil.urlProvider(QueryUrls.class).urlCreateExcelTemplate(ctx.getContainer(), getPublicSchemaName(), getName()); + url.addParameter("headerType", ColumnHeaderType.ImportField.name()); + try + { + if (getSampleType() != null && !getSampleType().getImportAliases().isEmpty()) + { + for (String aliasKey : getSampleType().getImportAliases().keySet()) + url.addParameter("includeColumn", aliasKey); + } + } + catch (IOException e) + {} + templates.add(Pair.of("Download Template", url.toString())); + return templates; + } + + @Override + public void overlayMetadata(String tableName, UserSchema schema, Collection errors) + { + if (SamplesSchema.SCHEMA_NAME.equals(schema.getName())) + { + Collection metadata = QueryService.get().findMetadataOverride(schema, SamplesSchema.SCHEMA_METADATA_NAME, false, false, errors, null); + if (null != metadata) + { + overlayMetadata(metadata, schema, errors); + } + } + super.overlayMetadata(tableName, schema, errors); + } + + static class SampleTypeAmountPrecisionDisplayColumn extends DataColumn + { + Unit typeUnit; + boolean applySampleTypePrecision = true; + + public SampleTypeAmountPrecisionDisplayColumn(ColumnInfo col, Unit typeUnit) { + super(col, false); + this.typeUnit = typeUnit; + this.applySampleTypePrecision = col.getFormat() == null; // only apply if no custom format is set by user + } + + @Override + public Object getDisplayValue(RenderContext ctx) + { + Object value = super.getDisplayValue(ctx); + if (this.applySampleTypePrecision && value != null) + { + int scale = this.typeUnit == null ? Quantity.DEFAULT_PRECISION_SCALE : this.typeUnit.getPrecisionScale(); + value = Precision.round(Double.valueOf(value.toString()), scale); + } + return value; + } + } +} From 28ff95145cd76e6f230fbeeafcde38cda1c4eac2 Mon Sep 17 00:00:00 2001 From: cnathe Date: Mon, 6 Oct 2025 13:30:44 -0500 Subject: [PATCH 3/3] CR feedback --- .../src/org/labkey/experiment/api/ExpMaterialTableImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index 15c26069d6e..2356cc65f8b 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -1832,8 +1832,8 @@ public void overlayMetadata(String tableName, UserSchema schema, Collection