Update Amounts and Units for better sorting and filtering behavior

api/src/org/labkey/api/ApiModule.java

@@ -82,7 +82,6 @@
 import org.labkey.api.data.dialect.JdbcMetaDataTest;
 import org.labkey.api.data.dialect.ParameterSubstitutionTest;
 import org.labkey.api.data.dialect.StandardDialectStringHandler;
-import org.labkey.api.data.measurement.Measurement;
 import org.labkey.api.dataiterator.CachingDataIterator;
 import org.labkey.api.dataiterator.DataIteratorUtil;
 import org.labkey.api.dataiterator.DiskCachingDataIterator;
@@ -411,8 +410,6 @@ public void registerServlets(ServletContext servletCtx)
             LimitedUser.TestCase.class,
             MarkableIterator.TestCase.class,
             MaterializedQueryHelper.TestCase.class,
-            Measurement.TestCase.class,
-            Measurement.Unit.TestCase.class,
             MemTracker.TestCase.class,
             ModuleContext.TestCase.class,
             ModuleDependencySorter.TestCase.class,

api/src/org/labkey/api/audit/AbstractAuditHandler.java

@@ -45,16 +45,17 @@ public void addSummaryAuditEvent(User user, Container c, TableInfo table, QueryS
 
     /**
      * Create a detailed audit record object so it can be recorded in the audit tables
-     * @param user making change
-     * @param c container containing auditable data
-     * @param tInfo Auditable tableInfo containing auditable record
-     * @param action being performed
-     * @param userComment Comment provided by the user explaining reason for change. NOTE: This value is generally not currently supported by many audit logging domains, and may be ignored.
-     * @param row map of new data values
-     * @param existingRow map of data values
+     * @param user           making change
+     * @param c              container containing auditable data
+     * @param tInfo          Auditable tableInfo containing auditable record
+     * @param action         being performed
+     * @param userComment    Comment provided by the user explaining reason for change. NOTE: This value is generally not currently supported by many audit logging domains, and may be ignored.
+     * @param row            map of new data values
+     * @param existingRow    map of data values
+     * @param providedValues map of values provided by the user before conversion (e.g., for quantity values)
      * @return DetailedAuditTypeEvent object describing audit record (NOTE: not committed to DB yet)
      */
-    protected abstract DetailedAuditTypeEvent createDetailedAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, @Nullable Map<String, Object> row, Map<String, Object> existingRow);
+    protected abstract DetailedAuditTypeEvent createDetailedAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, @Nullable Map<String, Object> row, Map<String, Object> existingRow, Map<String, Object> providedValues);
 
     /**
      * Allow for adding fields that may be present in the updated row but not represented in the original row
@@ -69,7 +70,7 @@ protected void addDetailedModifiedFields(Map<String, Object> originalRow, Map<St
     }
 
     @Override
-    public void addAuditEvent(User user, Container c, TableInfo table, @Nullable AuditBehaviorType auditType, @Nullable String userComment, QueryService.AuditAction action, List<Map<String, Object>> rows, @Nullable List<Map<String, Object>> existingRows, boolean useTransactionAuditCache)
+    public void addAuditEvent(User user, Container c, TableInfo table, @Nullable AuditBehaviorType auditType, @Nullable String userComment, QueryService.AuditAction action, List<Map<String, Object>> rows, @Nullable List<Map<String, Object>> existingRows, @Nullable List<Map<String, Object>> providedValues, boolean useTransactionAuditCache)
     {
         if (table.supportsAuditTracking())
         {
@@ -119,7 +120,8 @@ public void addAuditEvent(User user, Container c, TableInfo table, @Nullable Aud
                     {
                         Map<String, Object> row = rows.get(i);
                         Map<String, Object> existingRow = null == existingRows ? Collections.emptyMap() : existingRows.get(i);
-                        DetailedAuditTypeEvent event = createDetailedAuditRecord(user, c, auditConfigurable, action, userComment, row, existingRow);
+                        Map<String, Object> providedValueRow = null == providedValues || providedValues.size() <= i  ? null : providedValues.get(i);
+                        DetailedAuditTypeEvent event = createDetailedAuditRecord(user, c, auditConfigurable, action, userComment, row, existingRow, providedValueRow);
 
                         switch (action)
                         {

api/src/org/labkey/api/audit/AuditHandler.java

@@ -32,6 +32,9 @@
 
 public interface AuditHandler
 {
+    String PROVIDED_DATA_PREFIX = ":::provided:::";
+    String DELTA_PROVIDED_DATA_PREFIX = ":::delta_provided:::";
+
     void addSummaryAuditEvent(User user, Container c, TableInfo table, QueryService.AuditAction action, Integer dataRowCount, @Nullable AuditBehaviorType auditBehaviorType, @Nullable String userComment);
 
     default void addSummaryAuditEvent(User user, Container c, TableInfo table, QueryService.AuditAction action, Integer dataRowCount, @Nullable AuditBehaviorType auditBehaviorType, @Nullable String userComment, boolean skipAuditLevelCheck)
@@ -40,13 +43,20 @@ default void addSummaryAuditEvent(User user, Container c, TableInfo table, Query
     }
 
     void addAuditEvent(User user, Container c, TableInfo table, @Nullable AuditBehaviorType auditType, @Nullable String userComment, QueryService.AuditAction action,
-                       @Nullable List<Map<String, Object>> rows, @Nullable List<Map<String, Object>> existingRows, boolean useTransactionAuditCache);
+                       @Nullable List<Map<String, Object>> rows, @Nullable List<Map<String, Object>> existingRows, @Nullable List<Map<String, Object>> providedValues, boolean useTransactionAuditCache);
+
+    /* In the case of update the 'existingRows' is the 'before' version of the record. Caller is not expected to provide existingRows without rows. */
+    default void addAuditEvent(User user, Container c, TableInfo table, @Nullable AuditBehaviorType auditType, @Nullable String userComment, QueryService.AuditAction action,
+                               @Nullable List<Map<String, Object>> rows, @Nullable List<Map<String, Object>> existingRows)
+    {
+        addAuditEvent(user, c, table, auditType, userComment, action, rows, existingRows, null, false);
+    }
 
     /* In the case of update the 'existingRows' is the 'before' version of the record. Caller is not expected to provide existingRows without rows. */
     default void addAuditEvent(User user, Container c, TableInfo table, @Nullable AuditBehaviorType auditType, @Nullable String userComment, QueryService.AuditAction action,
-                       @Nullable List<Map<String, Object>> rows, @Nullable List<Map<String, Object>> existingRows)
+                               @Nullable List<Map<String, Object>> rows, @Nullable List<Map<String, Object>> existingRows, @Nullable List<Map<String, Object>> providedValues)
     {
-        addAuditEvent(user, c, table, auditType, userComment, action, rows, existingRows, false);
+        addAuditEvent(user, c, table, auditType, userComment, action, rows, existingRows, providedValues, false);
     }
 
 

api/src/org/labkey/api/audit/SampleTimelineAuditEvent.java

@@ -13,13 +13,22 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.labkey.api.audit.AuditHandler.DELTA_PROVIDED_DATA_PREFIX;
+import static org.labkey.api.audit.AuditHandler.PROVIDED_DATA_PREFIX;
+import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount;
+import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units;
+
 public class SampleTimelineAuditEvent extends DetailedAuditTypeEvent
 {
     public static final String EVENT_TYPE = "SampleTimelineEvent";
 
     public static final String SAMPLE_TIMELINE_EVENT_TYPE = "SampleTimelineEventType";
+    public static final String AMOUNT_AND_UNIT_UPGRADE_COMMENT = "Storage amount unit conversion to base unit during upgrade script.";
 
-    public static final Set<String> EXCLUDED_DETAIL_FIELDS = Set.of("AvailableAliquotVolume", "AvailableAliquotCount", "AliquotCount", "AliquotVolume", "AliquotUnit");
+    public static final Set<String> EXCLUDED_DETAIL_FIELDS = Set.of(
+            "AvailableAliquotVolume", "AvailableAliquotCount", "AliquotCount", "AliquotVolume", "AliquotUnit",
+            PROVIDED_DATA_PREFIX + StoredAmount.name(), PROVIDED_DATA_PREFIX + Units.name(),
+            DELTA_PROVIDED_DATA_PREFIX + StoredAmount.name() + DELTA_PROVIDED_DATA_PREFIX + Units.name());
 
     public enum SampleTimelineEventType
     {

api/src/org/labkey/api/data/EnumTableInfo.java

@@ -37,6 +37,7 @@
 public class EnumTableInfo<EnumType extends Enum<EnumType>> extends VirtualTable<UserSchema>
 {
     protected final Class<EnumType> _enum;
+    protected final EnumSet<EnumType> _enumSet;
     protected final EnumValueGetter<EnumType> _valueGetter;
     protected final EnumRowIdGetter<EnumType> _rowIdGetter;
     @Nullable protected String _schemaName;
@@ -93,6 +94,11 @@ public EnumTableInfo(Class<EnumType> e, UserSchema schema, EnumRowIdGetter<EnumT
         this(e, schema, EnumType::toString, rowIdGetter, rowIdPK, description);
     }
 
+    public EnumTableInfo(Class<EnumType> e, UserSchema schema, EnumValueGetter<EnumType> valueGetter, EnumRowIdGetter<EnumType> rowIdGetter, boolean rowIdPK, String description)
+    {
+        this(e, EnumSet.allOf(e), schema, valueGetter, rowIdGetter, rowIdPK, description);
+    }
+
     /**
      * Exposes an enum as a three column virtual table, using valueGetter to determine its value, rowIdGetter to determine rowId, and ordinal() as ordinal
      * @param e class of the enum
@@ -102,11 +108,12 @@ public EnumTableInfo(Class<EnumType> e, UserSchema schema, EnumRowIdGetter<EnumT
      * @param rowIdPK use the rowId as the key field, otherwise use value field
      * @param description a description of this table and its uses for display in the schema browser
      */
-    public EnumTableInfo(Class<EnumType> e, UserSchema schema, EnumValueGetter<EnumType> valueGetter, EnumRowIdGetter<EnumType> rowIdGetter, boolean rowIdPK, String description)
+    public EnumTableInfo(Class<EnumType> e, EnumSet<EnumType> enumSet, UserSchema schema, EnumValueGetter<EnumType> valueGetter, EnumRowIdGetter<EnumType> rowIdGetter, boolean rowIdPK, String description)
     {
         super(schema.getDbSchema(), e.getSimpleName(), schema);
         setDescription(description);
         _enum = e;
+        _enumSet = enumSet;
         _valueGetter = valueGetter;
         _rowIdGetter = rowIdGetter;
 
@@ -131,8 +138,7 @@ public SQLFragment getFromSQL()
         checkReadBeforeExecute();
         SQLFragment sql = new SQLFragment();
         String separator = "";
-        EnumSet<EnumType> enumSet = EnumSet.allOf(_enum);
-        for (EnumType e : enumSet)
+        for (EnumType e : _enumSet)
         {
             sql.append(separator);
             separator = " UNION ";

api/src/org/labkey/api/data/ImportAliasable.java

@@ -16,10 +16,12 @@
 package org.labkey.api.data;
 
 import org.labkey.api.collections.CaseInsensitiveHashMap;
+import org.labkey.api.collections.CaseInsensitiveHashSet;
 import org.labkey.api.exp.MvColumn;
 import org.labkey.api.view.template.PageConfig;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -45,6 +47,23 @@ public interface ImportAliasable
 
     class Helper
     {
+        /**
+         * Creates a set of the different possible aliases for import (name, label, import aliases) for a single column.
+         */
+        public static <T extends ImportAliasable> Set<String> createImportSet(T property)
+        {
+            if (property == null)
+                return Collections.emptySet();
+
+            Set<String> aliases = new CaseInsensitiveHashSet();
+            aliases.add(property.getName());
+            if (property.getLabel() != null)
+                aliases.add(property.getLabel());
+            if (!property.getImportAliasSet().isEmpty())
+                aliases.addAll(property.getImportAliasSet());
+            return aliases;
+        }
+
         /**
          * Creates a mapping of many different possible names (actual name, label/caption, property URI, etc).
          * for a column to the column itself. Useful to provide flexibility in how the data is labeled during imports.

api/src/org/labkey/api/data/TableInfo.java

@@ -154,7 +154,7 @@ public void addSummaryAuditEvent(User user, Container c, TableInfo table, QueryS
         }
 
         @Override
-        public void addAuditEvent(User user, Container c, TableInfo table, @Nullable AuditBehaviorType auditType, @Nullable String userComment, QueryService.AuditAction action, List<Map<String, Object>> rows, @Nullable List<Map<String, Object>> updatedRows, boolean useTransactionAuditCache)
+        public void addAuditEvent(User user, Container c, TableInfo table, @Nullable AuditBehaviorType auditType, @Nullable String userComment, QueryService.AuditAction action, List<Map<String, Object>> rows, @Nullable List<Map<String, Object>> updatedRows, @Nullable List<Map<String, Object>> providedValues, boolean useTransactionAuditCache)
         {
         }
     }

api/src/org/labkey/api/data/generator/DataGenerator.java

@@ -16,7 +16,6 @@
 import org.labkey.api.data.TableInfo;
 import org.labkey.api.data.TableSelector;
 import org.labkey.api.data.dialect.SqlDialect;
-import org.labkey.api.data.measurement.Measurement;
 import org.labkey.api.dataiterator.DataIteratorBuilder;
 import org.labkey.api.dataiterator.DetailedAuditLogDataIterator;
 import org.labkey.api.dataiterator.MapDataIterator;
@@ -34,6 +33,8 @@
 import org.labkey.api.exp.query.SamplesSchema;
 import org.labkey.api.gwt.client.AuditBehaviorType;
 import org.labkey.api.gwt.client.model.GWTPropertyDescriptor;
+import org.labkey.api.ontology.KindOfQuantity;
+import org.labkey.api.ontology.Unit;
 import org.labkey.api.pipeline.CancelledException;
 import org.labkey.api.pipeline.PipelineJob;
 import org.labkey.api.qc.DataState;
@@ -76,12 +77,7 @@ public class DataGenerator<T extends DataGenerator.Config> implements ContainerU
     private static final String SEARCH_FIELD_NAME = "Search";
     private static final String USED_FIELD_NAME = "Used";
     private static final List<String> UNITS = List.of("g", "mg", "kg", "uL", "mL", "L", "unit");
-    // TODO use KindOfQuantity
-    private static final Map<Measurement.Kind, List<String>> UNIT_KINDS = Map.of(
-            Measurement.Kind.Mass, List.of("g", "mg", "kg"),
-            Measurement.Kind.Count, List.of("units"),
-            Measurement.Kind.Volume, List.of("uL", "mL", "L")
-    );
+
     private static final List<String> LABEL_COLORS = new ArrayList<>();
     static {
         LABEL_COLORS.add("#800000"); // maroon,
@@ -846,8 +842,8 @@ private static String getUnitsValue(ExpSampleType sampleType)
         {
             if (!StringUtils.isEmpty(sampleType.getMetricUnit()))
             {
-                Measurement.Unit unit = Measurement.Unit.getUnit(sampleType.getMetricUnit());
-                units = randomIndex(UNIT_KINDS.get(unit.getKind()));
+                Unit unit = Unit.fromName(sampleType.getMetricUnit());
+                units = randomIndex(unit.getKindOfQuantity().getCommonUnits()).name();
             }
             else
                 units = randomIndex(UNITS);