Issue 53567: Improve "Alias" column performance

api/src/org/labkey/api/audit/AuditHandler.java

@@ -87,7 +87,7 @@ static Pair<Map<String, Object>, Map<String, Object>> getOldAndNewRecordForMerge
             String key = entry.getKey();
             // getDatasetRows() (at least) should return key==column.getName(), expect getColumn(name) to work
             ColumnInfo col = null==table ? null : table.getColumn(key);
-            if (col != null && col.getFk() instanceof MultiValuedForeignKey)
+            if (col != null && (col.isMultiValued() || col.getFk() instanceof MultiValuedForeignKey))
                 isMultiValued = true;
 
             String nameFromAlias = key;
@@ -99,7 +99,7 @@ static Pair<Map<String, Object>, Map<String, Object>> getOldAndNewRecordForMerge
 
                 if (aliasColumn != null)
                 {
-                    if (aliasColumn.getFk() != null && aliasColumn.getFk() instanceof MultiValuedForeignKey)
+                    if (aliasColumn.getFk() != null && (aliasColumn.isMultiValued() || aliasColumn.getFk() instanceof MultiValuedForeignKey))
                         isMultiValued = true;
                     nameFromAlias = aliasColumn.getName();
                 }

api/src/org/labkey/api/data/AbstractWrappedColumnInfo.java

@@ -842,4 +842,10 @@ public boolean isScannable()
     {
         return delegate.isScannable();
     }
+
+    @Override
+    public boolean isMultiValued()
+    {
+        return delegate.isMultiValued();
+    }
 }

api/src/org/labkey/api/data/BaseColumnInfo.java

@@ -97,6 +97,7 @@ public class BaseColumnInfo extends ColumnRenderPropertiesImpl implements Mutabl
     private String _jdbcDefaultValue = null;  // TODO: Merge with defaultValue, see #17646
     private boolean _isAutoIncrement = false;
     private boolean _hasDbSequence = false;
+    private boolean _isMultiValued = false;
     private boolean _isRootDbSequence = false;
     private boolean _isKeyField = false;
     private boolean _isReadOnly = false;
@@ -324,6 +325,7 @@ public void copyAttributesFrom(ColumnInfo col)
 
         // and the remaining
         setUserEditable(col.isUserEditable());
+        setIsMultiValued(col.isMultiValued());
         setNullable(col.isNullable());
         setRequired(col.isRequiredSet());
         setAutoIncrement(col.isAutoIncrement());
@@ -445,6 +447,7 @@ public void setExtraAttributesFrom(ColumnInfo col)
 
         setDerivationDataScope(col.getDerivationDataScope());
         setScannable(col.isScannable());
+        setIsMultiValued(col.isMultiValued());
     }
 
     /*
@@ -988,6 +991,17 @@ public void setIsRootDbSequence(boolean isRootDbSequence)
         _isRootDbSequence = isRootDbSequence;
     }
 
+    public boolean isMultiValued()
+    {
+        return _isMultiValued;
+    }
+
+    public void setIsMultiValued(boolean isMultiValued)
+    {
+        checkLocked();
+        _isMultiValued = isMultiValued;
+    }
+
     @Override
     public boolean isReadOnly()
     {

api/src/org/labkey/api/data/ColumnInfo.java

@@ -290,6 +290,8 @@ default boolean isSortable()
      */
     boolean isKeyField();
 
+    boolean isMultiValued();
+
     @Override
     boolean isMvEnabled();
 

api/src/org/labkey/api/data/MultiValuedLookupColumn.java

@@ -211,4 +211,10 @@ protected SQLFragment getAggregateFunction(SQLFragment sql)
         // Can't sort because we need to make sure that all the multi-value columns come back in the same order
         return getSqlDialect().getGroupConcat(sql, false, false, new SQLFragment().appendStringLiteral(MultiValuedRenderContext.VALUE_DELIMITER, getSqlDialect()), true);
     }
+
+    @Override
+    public boolean isMultiValued()
+    {
+        return true;
+    }
 }

api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java

@@ -487,36 +487,37 @@ public SQLFragment getGroupConcat(SQLFragment sql, boolean distinct, boolean sor
     {
         // Sort function might not exist in external datasource; skip that syntax if not
         boolean useSortFunction = sorted && _arraySortFunctionExists.get();
+        SQLFragment result = new SQLFragment();
 
-        // TODO: Use "string_agg()" in place of array_to_string(array_agg())?
-        SQLFragment result = new SQLFragment("array_to_string(");
         if (useSortFunction)
         {
+            result.append("array_to_string(");
             result.append("core.sort(");   // TODO: Switch to use ORDER BY option inside array aggregate instead of our custom function
+            result.append("array_agg(");
         }
-        result.append("array_agg(");
-        if (distinct)
+        else
         {
-            result.append("DISTINCT ");
+            result.append("string_agg(");
         }
-        if (includeNulls)
+
+        if (distinct)
         {
-            result.append("COALESCE(CAST(");
+            result.append("DISTINCT ");
         }
         result.append(sql);
 
         if (includeNulls)
         {
-            result.append(" AS VARCHAR), '')");
+            result.append("::text");
         }
-        result.append(")");
         if (useSortFunction)
         {
-            result.append(")");
+            result.append(")"); // array_agg
+            result.append(")"); // core.sort
         }
         result.append(", ");
         result.append(delimiterSQL);
-        result.append(")");
+        result.append(")"); // array_to_string | string_agg
 
         return result;
     }

api/src/org/labkey/api/data/generator/DataGenerator.java

@@ -450,7 +450,7 @@ public int generateAliquots(ExpSampleType sampleType, QueryUpdateService svc, in
         {
             checkAlive(_job);
             List<Map<String, Object>> parents = getRandomSamples(sampleType, Math.min(10, Math.max(quantity, quantity / 100)));
-            numGenerated = generateAliquotsForParents(parents, svc, quantity - totalAliquots, 0, 1, randomInt(1, _config.getMaxGenerations()), sampleStatuses);
+            numGenerated = generateAliquotsForParents(sampleType, parents, svc, quantity - totalAliquots, 0, 1, randomInt(1, _config.getMaxGenerations()), sampleStatuses);
             totalAliquots += numGenerated;
             _log.info("... " + totalAliquots);
             iterations++;
@@ -463,7 +463,7 @@ public int generateAliquots(ExpSampleType sampleType, QueryUpdateService svc, in
         return totalAliquots;
     }
 
-    private int generateAliquotsForParents(List<Map<String, Object>> parents, QueryUpdateService svc, int quantity, int numGenerated, int generation, int maxGenerations, List<Long> sampleStatuses) throws SQLException, BatchValidationException, QueryUpdateServiceException, DuplicateKeyException
+    private int generateAliquotsForParents(ExpSampleType sampleType, List<Map<String, Object>> parents, QueryUpdateService svc, int quantity, int numGenerated, int generation, int maxGenerations, List<Long> sampleStatuses) throws SQLException, BatchValidationException, QueryUpdateServiceException, DuplicateKeyException
     {
         int generatedCount = 0;
         List<Map<String, Object>> aliquots = new ArrayList<>();
@@ -492,6 +492,7 @@ private int generateAliquotsForParents(List<Map<String, Object>> parents, QueryU
                 if (randomInt(0, 10) == 5) // set amount for 10%
                 {
                     row.put("StoredAmount", p + (i % 15) * (1.2));
+                    row.put("Units", getUnitsValue(sampleType));
                 }
                 rows.add(row);
             }
@@ -511,7 +512,7 @@ private int generateAliquotsForParents(List<Map<String, Object>> parents, QueryU
         {
             List<Map<String, Object>> nextParents = aliquots.isEmpty() ? parents : aliquots;
             if (!nextParents.isEmpty())
-                generatedCount += generateAliquotsForParents(nextParents.subList(randomInt(0, nextParents.size() / 2), randomInt(nextParents.size() / 2, nextParents.size())), svc, quantity - generatedCount, numGenerated + generatedCount, generation + 1, maxGenerations, sampleStatuses);
+                generatedCount += generateAliquotsForParents(sampleType, nextParents.subList(randomInt(0, nextParents.size() / 2), randomInt(nextParents.size() / 2, nextParents.size())), svc, quantity - generatedCount, numGenerated + generatedCount, generation + 1, maxGenerations, sampleStatuses);
         }
         return generatedCount;
     }
@@ -836,12 +837,10 @@ protected void addDomainProperties(List<GWTPropertyDescriptor> props, int numFie
 
     private static String getUnitsValue(ExpSampleType sampleType)
     {
+        Unit sampleTypeUnit = Unit.fromName(sampleType.getMetricUnit());
         String units;
-        if (!StringUtils.isEmpty(sampleType.getMetricUnit()))
-        {
-            Unit unit = Unit.fromName(sampleType.getMetricUnit());
-            units = randomIndex(unit.getKindOfQuantity().getCommonUnits()).name();
-        }
+        if (sampleTypeUnit != null)
+            units = randomIndex(sampleTypeUnit.getKindOfQuantity().getCommonUnits()).name();
         else
             units = randomIndex(UNITS);
 
@@ -854,10 +853,7 @@ public static Map<String, Object> createSampleRow(ExpSampleType sampleType, int
         if (randomInt(0, 4) < 3) // set amount for 75% - TODO parameterize
         {
             row.put("StoredAmount", randomDouble(0, 100));
-            // add a unit for some percentage
-            String units = getUnitsValue(sampleType);
-            if (units != null)
-                row.put("Units", units);
+            row.put("Units", getUnitsValue(sampleType));
         }
         return row;
     }
@@ -897,13 +893,31 @@ protected List<Map<String, Object>> createRows(int numRows, Domain domain)
     protected List<Map<String, Object>> createSampleRows(int numRows, ExpSampleType sampleType)
     {
         List<Map<String, Object>> rows = new ArrayList<>();
+        int samplesWithAliases = Math.round(numRows * _config.getPctAlias());
         for (int i = 1; i <= numRows; i++)
         {
-            rows.add(createSampleRow(sampleType, i));
+            Map<String, Object> row = createSampleRow(sampleType, i);
+            if (i < samplesWithAliases)
+            {
+                row.put("Alias", randomAlias("alias_"));
+            }
+            rows.add(row);
         }
         return rows;
     }
 
+    private String randomAlias(String prefix)
+    {
+        StringBuilder sb = new StringBuilder();
+        for (int i = 1; i <= randomInt(1, _config._maxAliases); i++)
+        {
+            if (!sb.isEmpty())
+                sb.append(", ");
+            sb.append(prefix).append(i);
+        }
+        return sb.toString();
+    }
+
     public static String randomDate()
     {
         var startDate = new Date(112 /* 2012 */, Calendar.JANUARY, 1);
@@ -962,6 +976,8 @@ public static class Config
         public static final String MIN_DATA_CLASS_PARENT_TYPES_PER_SAMPLE = "minDataClassParentTypesPerSample";
         public static final String MAX_DATA_CLASS_PARENT_TYPES_PER_SAMPLE = "maxDataClassParentTypesPerSample";
         public static final String AUDIT_BEHAVIOR_TYPE = "auditBehaviorType";
+        public static final String PCT_ALIAS = "percentWithAliases";
+        public static final String MAX_ALIAS = "maxAliases";
 
         int _numFolders;
         int _numSampleTypes;
@@ -977,6 +993,8 @@ public static class Config
         int _maxChildrenPerParent;
         int _minFields;
         int _maxFields;
+        float _pctAlias;
+        int _maxAliases;
         List<String> _sampleTypeNames;
         List<String> _dataClassParents;
         List<String> _sampleTypeParents;
@@ -1010,6 +1028,9 @@ public Config(Properties properties)
             _maxAliquotsPerParent = Integer.parseInt(properties.getProperty(MAX_ALIQUOTS_PER_SAMPLE, "0"));
             _maxChildrenPerParent = Integer.parseInt(properties.getProperty(MAX_CHILDREN_PER_PARENT, "1"));
 
+            _pctAlias = Float.parseFloat(properties.getProperty(PCT_ALIAS, "0.0"));
+            _maxAliases = Integer.parseInt(properties.getProperty(MAX_ALIAS, "1"));
+
             _minFields = Integer.parseInt(properties.getProperty(MIN_NUM_FIELDS, "1"));
             _maxFields = Math.max(Integer.parseInt(properties.getProperty(MAX_NUM_FIELDS, "1")), _minFields);
 
@@ -1235,6 +1256,26 @@ public void setAuditBehaviorType(AuditBehaviorType auditBehaviorType)
         {
             _auditBehaviorType = auditBehaviorType;
         }
+
+        public float getPctAlias()
+        {
+            return _pctAlias;
+        }
+
+        public void setPctAlias(float pctAlias)
+        {
+            _pctAlias = pctAlias;
+        }
+
+        public int getMaxAliases()
+        {
+            return _maxAliases;
+        }
+
+        public void setMaxAliases(int maxAliases)
+        {
+            _maxAliases = maxAliases;
+        }
     }
 
     public interface DataGenerationDriver

api/src/org/labkey/api/dataiterator/CoerceDataIterator.java

@@ -65,7 +65,7 @@ void init(TableInfo target, boolean useImportAliases, boolean outputAllColumns)
                 seen.add(to.getName());
                 if (to.getPropertyType() == PropertyType.ATTACHMENT || to.getPropertyType() == PropertyType.FILE_LINK)
                     addColumn(to, i);
-                else if (to.getFk() instanceof MultiValuedForeignKey)
+                else if (to.isMultiValued() || to.getFk() instanceof MultiValuedForeignKey)
                     addColumn(to.getName(), i); // pass-through multi-value columns -- converting will stringify a collection
                 else
                     addConvertColumn(to.getName(), i, to.getJdbcType(), to.getFk(), RemapMissingBehavior.Error, true);

api/src/org/labkey/api/dataiterator/SimpleTranslator.java

@@ -1356,7 +1356,7 @@ private SimpleConvertColumn createConvertColumn(@NotNull ColumnInfo col, int fro
             c = new RemappingConvertColumn(c, fromIndex, col, missing, true, lookupResolutionType);
         }
 
-        boolean multiValue = fk instanceof MultiValuedForeignKey;
+        boolean multiValue = col.isMultiValued() || fk instanceof MultiValuedForeignKey;
         if (multiValue)
         {
             // convert input into Collection of jdbcType values

api/src/org/labkey/api/query/AbstractQueryUpdateService.java

@@ -716,6 +716,39 @@ public List<Map<String, Object>> insertRows(User user, Container container, List
         return null;
     }
 
+    protected Object coerceTypesValue(ColumnInfo col, Map<String, Object> providedValues, String key, Object value)
+    {
+        if (col != null && value != null &&
+            !col.getJavaObjectClass().isInstance(value) &&
+            !(value instanceof AttachmentFile) &&
+            !(value instanceof MultipartFile) &&
+            !(value instanceof String[]) &&
+            !(col.isMultiValued() || col.getFk() instanceof MultiValuedForeignKey))
+        {
+            try
+            {
+                if (PropertyType.FILE_LINK.equals(col.getPropertyType()))
+                    value = ExpDataFileConverter.convert(value);
+                else if (col.getKindOfQuantity() != null)
+                {
+                    providedValues.put(key, value);
+                    value = Quantity.convert(value, col.getKindOfQuantity().getStorageUnit());
+                }
+                else
+                    value = col.getConvertFn().apply(value);
+            }
+            catch (ConvertHelper.FileConversionException e)
+            {
+                throw e;
+            }
+            catch (ConversionException e)
+            {
+                // That's OK, the transformation script may be able to fix up the value before it gets inserted
+            }
+        }
+
+        return value;
+    }
 
     /** Attempt to make the passed in types match the expected types so the script doesn't have to do the conversion */
     @Deprecated
@@ -726,42 +759,13 @@ protected Map<String, Object> coerceTypes(Map<String, Object> row, Map<String, O
         for (Map.Entry<String, Object> entry : row.entrySet())
         {
             ColumnInfo col = columnMap.get(entry.getKey());
-
-            Object value = entry.getValue();
-            if (col != null && value != null &&
-                    !col.getJavaObjectClass().isInstance(value) &&
-                    !(value instanceof AttachmentFile) &&
-                    !(value instanceof MultipartFile) &&
-                    !(value instanceof String[]) &&
-                    !(col.getFk() instanceof MultiValuedForeignKey))
-            {
-                try
-                {
-                    if (PropertyType.FILE_LINK.equals(col.getPropertyType()))
-                        value = ExpDataFileConverter.convert(value);
-                    else if (col.getKindOfQuantity() != null)
-                    {
-                        providedValues.put(entry.getKey(), value);
-                        value = Quantity.convert(value, col.getKindOfQuantity().getStorageUnit());
-                    }
-                    else
-                        value = col.getConvertFn().apply(value);
-                }
-                catch (ConvertHelper.FileConversionException e)
-                {
-                    throw e;
-                }
-                catch (ConversionException e)
-                {
-                    // That's OK, the transformation script may be able to fix up the value before it gets inserted
-                }
-            }
+            Object value = coerceTypesValue(col, providedValues, entry.getKey(), entry.getValue());
             result.put(entry.getKey(), value);
         }
+
         return result;
     }
 
-
     protected abstract Map<String, Object> updateRow(User user, Container container, Map<String, Object> row, @NotNull Map<String, Object> oldRow, @Nullable Map<Enum, Object> configParameters)
             throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException;