LK R&D LKSM - Update default audit level

api/gwtsrc/org/labkey/api/gwt/client/AuditBehaviorType.java

@@ -15,25 +15,43 @@
  */
 package org.labkey.api.gwt.client;
 
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
 /**
  * User: klum
  * Date: 10/17/12
  */
 public enum AuditBehaviorType
 {
-    NONE("None"),
-    DETAILED("Detailed"),
-    SUMMARY("Summary");
+    NONE("None", 1),
+    DETAILED("Detailed", 3),
+    SUMMARY("Summary", 2);
 
     private final String _label;
+    private final int _priority;
 
-    AuditBehaviorType(String label)
+    AuditBehaviorType(String label, int priority)
     {
         _label = label;
+        _priority = priority;
     }
 
     public String getLabel()
     {
         return _label;
     }
+
+    public int getPriority()
+    {
+        return _priority;
+    }
+
+    public static AuditBehaviorType getEffectiveAuditLevel(@Nullable AuditBehaviorType apiOverride, @NotNull AuditBehaviorType tableAuditBehaviorType)
+    {
+        if (apiOverride == null || apiOverride._priority < tableAuditBehaviorType._priority)
+            return tableAuditBehaviorType;
+
+        return apiOverride;
+    }
 }

api/src/org/labkey/api/audit/AbstractAuditHandler.java

@@ -22,13 +22,19 @@ public abstract class AbstractAuditHandler implements AuditHandler
 
     @Override
     public void addSummaryAuditEvent(User user, Container c, TableInfo table, QueryService.AuditAction action, Integer dataRowCount, @Nullable AuditBehaviorType auditBehaviorType, @Nullable String userComment)
+    {
+        addSummaryAuditEvent(user, c, table, action, dataRowCount, auditBehaviorType, userComment, false);
+    }
+
+    @Override
+    public void addSummaryAuditEvent(User user, Container c, TableInfo table, QueryService.AuditAction action, Integer dataRowCount, @Nullable AuditBehaviorType auditBehaviorType, @Nullable String userComment, boolean skipAuditLevelCheck)
     {
         if (table.supportsAuditTracking())
         {
             AuditConfigurable auditConfigurable = (AuditConfigurable) table;
-            AuditBehaviorType auditType = auditBehaviorType == null ? auditConfigurable.getAuditBehavior() : auditBehaviorType;
+            AuditBehaviorType auditType = auditConfigurable.getEffectiveAuditBehavior(auditBehaviorType);
 
-            if (auditType == SUMMARY)
+            if (auditType == SUMMARY || skipAuditLevelCheck)
             {
                 AuditTypeEvent event = createSummaryAuditRecord(user, c, auditConfigurable, action, userComment, dataRowCount, null);
 
@@ -68,7 +74,7 @@ public void addAuditEvent(User user, Container c, TableInfo table, @Nullable Aud
         if (table.supportsAuditTracking())
         {
             AuditConfigurable auditConfigurable = (AuditConfigurable)table;
-            auditType = auditConfigurable.getAuditBehavior(auditType);
+            auditType = auditConfigurable.getEffectiveAuditBehavior(auditType);
 
             // Truncate audit event doesn't accept any params
             if (action == QueryService.AuditAction.TRUNCATE)
@@ -99,6 +105,8 @@ public void addAuditEvent(User user, Container c, TableInfo table, @Nullable Aud
                     AuditTypeEvent event = createSummaryAuditRecord(user, c, auditConfigurable, action, userComment, rows.size(), rows.get(0));
 
                     AuditLogService.get().addEvent(user, event);
+
+                    return;
                 }
                 case DETAILED:
                 {

api/src/org/labkey/api/audit/AuditHandler.java

@@ -33,6 +33,11 @@ public interface AuditHandler
 {
     void addSummaryAuditEvent(User user, Container c, TableInfo table, QueryService.AuditAction action, Integer dataRowCount, @Nullable AuditBehaviorType auditBehaviorType, @Nullable String userComment);
 
+    default void addSummaryAuditEvent(User user, Container c, TableInfo table, QueryService.AuditAction action, Integer dataRowCount, @Nullable AuditBehaviorType auditBehaviorType, @Nullable String userComment, boolean skipAuditLevelCheck)
+    {
+        addSummaryAuditEvent(user, c, table, action, dataRowCount, auditBehaviorType, userComment);
+    }
+
     void addAuditEvent(User user, Container c, TableInfo table, @Nullable AuditBehaviorType auditType, @Nullable String userComment, QueryService.AuditAction action,
                        @Nullable List<Map<String, Object>> rows, @Nullable List<Map<String, Object>> existingRows, boolean useTransactionAuditCache);
 

api/src/org/labkey/api/data/AbstractTableInfo.java

@@ -280,6 +280,12 @@ public AbstractTableInfo(DbSchema schema, String name)
         MemTracker.getInstance().put(this);
     }
 
+    @NotNull
+    @Override
+    public AuditBehaviorType getDefaultAuditBehavior()
+    {
+        return _auditBehaviorType;
+    }
 
     public void afterConstruct()
     {
@@ -1466,7 +1472,6 @@ else if (xmlColumn.isSetWrappedColumnName() && isNotBlank(xmlColumn.getWrappedCo
                 catch (QueryParseException qpe)
                 {
                     warnings.add(qpe.setFieldName(xmlColumn.getColumnName()));
-                    continue;
                 }
             }
 
@@ -1487,7 +1492,16 @@ else if (!calculatedFieldKeys.isEmpty())
         if (xmlTable.getAuditLogging() != null)
         {
             AuditType.Enum auditBehavior = xmlTable.getAuditLogging();
-            setAuditBehavior(AuditBehaviorType.valueOf(auditBehavior.toString()));
+            try
+            {
+                AuditBehaviorType auditBehaviorType = AuditBehaviorType.valueOf(auditBehavior.toString());
+                setAuditBehavior(auditBehaviorType);
+            }
+            catch (IllegalArgumentException ignore)
+            {
+                warnings.add(new QueryParseWarning("Invalid AuditLogging: " + auditBehavior,null,0,0));
+            }
+
         }
 
         _warnings.addAll(warnings);
@@ -2000,12 +2014,6 @@ public void setAuditBehavior(AuditBehaviorType type)
         _xmlAuditBehaviorType = type;
     }
 
-    @Override
-    public AuditBehaviorType getAuditBehavior()
-    {
-        return _auditBehaviorType;
-    }
-
     @Override
     public AuditBehaviorType getXmlAuditBehaviorType()
     {

api/src/org/labkey/api/data/SchemaTableInfo.java

@@ -16,6 +16,7 @@
 
 package org.labkey.api.data;
 
+import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.labkey.api.cache.CacheManager;
@@ -51,6 +52,7 @@
 import org.labkey.api.util.StringExpression;
 import org.labkey.api.util.StringExpressionFactory;
 import org.labkey.api.util.URLHelper;
+import org.labkey.api.util.logging.LogHelper;
 import org.labkey.api.view.ActionURL;
 import org.labkey.api.view.ViewContext;
 import org.labkey.data.xml.AuditType;
@@ -112,6 +114,8 @@ public class SchemaTableInfo implements TableInfo, UpdateableTableInfo, AuditCon
 
     protected boolean _autoLoadMetaData = true;      // TODO: Remove this? DatasetSchemaTableInfo is the only user of this.
 
+    private static final Logger LOG = LogHelper.getLogger(SchemaTableInfo.class, "TableInfo construction");
+
     public SchemaTableInfo(DbSchema parentSchema, DatabaseTableType tableType, String tableName, String metaDataName, SQLFragment selectName)
     {
         this(parentSchema, tableType, tableName, metaDataName, selectName, null);
@@ -527,14 +531,15 @@ public boolean supportsAuditTracking()
     }
 
     @Override
-    public void setAuditBehavior(AuditBehaviorType type)
+    public void setAuditBehavior(@NotNull AuditBehaviorType type)
      {
          _auditBehaviorType = type;
          _xmlAuditBehaviorType = type;
      }
 
-    @Override
-    public AuditBehaviorType getAuditBehavior()
+     @Override
+     @NotNull
+     public AuditBehaviorType getDefaultAuditBehavior()
      {
          return _auditBehaviorType;
      }
@@ -629,7 +634,15 @@ public void loadTablePropertiesFromXml(TableType xmlTable, boolean dontSetName)
         if (xmlTable.getAuditLogging() != null)
         {
             AuditType.Enum auditBehavior = xmlTable.getAuditLogging();
-            setAuditBehavior(AuditBehaviorType.valueOf(auditBehavior.toString()));
+            try
+            {
+                AuditBehaviorType auditBehaviorType = AuditBehaviorType.valueOf(auditBehavior.toString());
+                setAuditBehavior(auditBehaviorType);
+            }
+            catch (IllegalArgumentException ignore)
+            {
+                LOG.warn("Invalid AuditLogging: " + auditBehavior);
+            }
         }
 
         // Stash so we can overlay ColumnInfo properties later

api/src/org/labkey/api/data/TableInfo.java

@@ -161,7 +161,7 @@ public void addAuditEvent(User user, Container c, TableInfo table, @Nullable Aud
 
     default AuditHandler getAuditHandler(@Nullable AuditBehaviorType auditBehaviorOverride)
     {
-        if (!supportsAuditTracking() || auditBehaviorOverride == AuditBehaviorType.NONE || (auditBehaviorOverride == null && getAuditBehavior() == AuditBehaviorType.NONE))
+        if (!supportsAuditTracking() || getEffectiveAuditBehavior(auditBehaviorOverride) == AuditBehaviorType.NONE)
             return new _DoNothingAuditHandler();
         return QueryService.get().getDefaultAuditHandler();
     }
@@ -662,43 +662,46 @@ void fireRowTrigger(Container c, User user, TriggerType type, boolean before, in
     Set<ColumnInfo> getAllInvolvedColumns(Collection<ColumnInfo> selectColumns);
 
     /** see interface AuditConfigurable */
-    default AuditBehaviorType getAuditBehavior()
+    @NotNull
+    default AuditBehaviorType getDefaultAuditBehavior()
     {
         return AuditBehaviorType.NONE;
     }
 
+    @NotNull
+    default AuditBehaviorType getEffectiveAuditBehavior()
+    {
+        return getXmlAuditBehaviorType() == null ? getDefaultAuditBehavior() : getXmlAuditBehaviorType();
+    }
+
     /**
      * Retrieves the audit behavior for this table taking into account, in order of precedence:
-     *  - the setting from the XML file (always returned if there is a value set)
      *  - the override value provided
+     *  - the setting from the XML file (always returned if there is a value set)
      *  - the value supplied by this table's implementation
      *
-     * @param overrideValue value used to override the behavior type provided by the table implementation
+     * @param apiOverride value used to override the behavior type provided by the table implementation
      * @return audit behavior for this table
      */
-    default AuditBehaviorType getAuditBehavior(@Nullable AuditBehaviorType overrideValue)
+    @NotNull
+    default AuditBehaviorType getEffectiveAuditBehavior(@Nullable AuditBehaviorType apiOverride)
     {
-        AuditBehaviorType type = getXmlAuditBehaviorType();
-        if (type != null)
-            return type;
-        if (overrideValue != null)
-            return overrideValue;
-        return getAuditBehavior();
+        return AuditBehaviorType.getEffectiveAuditLevel(apiOverride, getEffectiveAuditBehavior());
     }
 
-    default AuditBehaviorType getAuditBehavior(@Nullable String overrideValue)
+    default AuditBehaviorType getEffectiveAuditBehavior(@Nullable String apiOverrideValue)
     {
-        if (overrideValue != null)
+        if (apiOverrideValue != null)
         {
             try
             {
-                return getAuditBehavior(AuditBehaviorType.valueOf(overrideValue));
+                return getEffectiveAuditBehavior(AuditBehaviorType.valueOf(apiOverrideValue));
             }
             catch (IllegalArgumentException ignored)
             {
             }
         }
-        return getAuditBehavior();
+        return getEffectiveAuditBehavior();
     }
 
     /* Can be used to distinguish AuditBehaviorType.NONE vs absent xml audit config */

api/src/org/labkey/api/dataiterator/DetailedAuditLogDataIterator.java

@@ -70,9 +70,10 @@ protected DetailedAuditLogDataIterator(DataIterator data, DataIteratorContext co
         _auditAction = auditAction;
         _auditHandler = table.getAuditHandler(DETAILED);
 
-        assert DETAILED == table.getAuditBehavior() || DETAILED == context.getConfigParameter(AuditConfigs.AuditBehavior);
+        assert DETAILED == table.getEffectiveAuditBehavior((AuditBehaviorType) context.getConfigParameter(AuditConfigs.AuditBehavior));
         assert !context.getInsertOption().mergeRows || _data.supportsGetExistingRecord();
         assert !context.getConfigParameterBoolean(QueryUpdateService.ConfigParameters.BulkLoad);
+        assert !context.getConfigParameterBoolean(QueryUpdateService.ConfigParameters.ByPassAudit);
 
         _existingRows = _data.supportsGetExistingRecord() ? new ArrayList<>() : null;
     }
@@ -129,10 +130,10 @@ public static DataIteratorBuilder getDataIteratorBuilder(TableInfo queryTable, @
         {
             AuditBehaviorType auditType = AuditBehaviorType.NONE;
             if (queryTable.supportsAuditTracking())
-                auditType = queryTable.getAuditBehavior((AuditBehaviorType) context.getConfigParameter(AuditConfigs.AuditBehavior));
+                auditType = queryTable.getEffectiveAuditBehavior((AuditBehaviorType) context.getConfigParameter(AuditConfigs.AuditBehavior));
 
             // Detailed auditing and not set to bulk load in ETL
-            if (auditType == DETAILED && !context.getConfigParameterBoolean(QueryUpdateService.ConfigParameters.BulkLoad))
+            if (auditType == DETAILED && !context.getConfigParameterBoolean(QueryUpdateService.ConfigParameters.BulkLoad) && !context.getConfigParameterBoolean(QueryUpdateService.ConfigParameters.ByPassAudit))
             {
                 DataIterator it = builder.getDataIterator(context);
                 DataIterator in = DataIteratorUtil.wrapMap(it, true);

api/src/org/labkey/api/dataiterator/ExistingRecordDataIterator.java

@@ -231,7 +231,7 @@ public static DataIteratorBuilder createBuilder(DataIteratorBuilder dib, TableIn
             {
                 AuditBehaviorType auditType = AuditBehaviorType.NONE;
                 if (target.supportsAuditTracking())
-                    auditType = target.getAuditBehavior((AuditBehaviorType) context.getConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior));
+                    auditType = target.getEffectiveAuditBehavior((AuditBehaviorType) context.getConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior));
                 boolean detailed = auditType == DETAILED;
                 if (useGetRows)
                     return new ExistingDataIteratorsGetRows(new CachingDataIterator(di), target, keys, sharedKeys, context, detailed);

api/src/org/labkey/api/exp/query/ExpSchema.java

@@ -440,16 +440,14 @@ public Set<SampleTypeService.SampleOperations> getPermittedOps()
      */
     public enum DataClassCategoryType
     {
-        registry(null, null),
-        media(null, null),
-        sources(AuditBehaviorType.DETAILED, ADDITIONAL_SOURCES_AUDIT_FIELDS);
+        registry(null),
+        media(null),
+        sources(ADDITIONAL_SOURCES_AUDIT_FIELDS);
 
-        public final AuditBehaviorType defaultBehavior;
         public final Set<String> additionalAuditFields;
 
-        DataClassCategoryType(@Nullable AuditBehaviorType defaultBehavior, @Nullable Set<String> addlAuditFields)
+        DataClassCategoryType(@Nullable Set<String> addlAuditFields)
         {
-            this.defaultBehavior = defaultBehavior;
             this.additionalAuditFields = addlAuditFields;
         }
 

api/src/org/labkey/api/query/AbstractQueryImportAction.java

@@ -435,9 +435,11 @@ public final ApiResponse _execute(FORM form, BindException errors) throws Except
 
         _useAsync = Boolean.valueOf(getParam(Params.useAsync)); // useAsync will save import file to pipeline root as well as run import in a background job
 
+        boolean isCrossTypeImport = getOptionParamsMap().getOrDefault(AbstractQueryImportAction.Params.crossTypeImport, false);
+
         // Check first if the audit behavior has been defined for the table either in code or through XML.
         // If not defined there, check for the audit behavior defined in the action form (getAuditBehaviorType()).
-        AuditBehaviorType behaviorType = (_target != null) ? _target.getAuditBehavior(getAuditBehaviorType()) : getAuditBehaviorType();
+        AuditBehaviorType behaviorType = (_target != null) ? _target.getEffectiveAuditBehavior(getAuditBehaviorType()) : getAuditBehaviorType();
 
         try
         {
@@ -630,8 +632,8 @@ else if (!dataFileDir.exists())
             configureLoader(loader);
 
             TransactionAuditProvider.TransactionAuditEvent auditEvent = null;
-            if (behaviorType != null && behaviorType != AuditBehaviorType.NONE)
-                auditEvent = createTransactionAuditEvent(getContainer(), QueryService.AuditAction.INSERT);
+            if (isCrossTypeImport || (behaviorType != null && behaviorType != AuditBehaviorType.NONE))
+                auditEvent = createTransactionAuditEvent(getContainer(), _insertOption.auditAction);
 
             int rowCount = importData(loader, file, originalName, ve, behaviorType, auditEvent, _auditUserComment);
 

api/src/org/labkey/api/query/AbstractQueryUpdateService.java

@@ -393,9 +393,18 @@ protected int _importRowsUsingDIB(User user, Container container, DataIteratorBu
             return 0;
         else
         {
-            AuditBehaviorType auditType = (AuditBehaviorType) context.getConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior);
-            String auditUserComment = (String) context.getConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment);
-            getQueryTable().getAuditHandler(auditType).addSummaryAuditEvent(user, container, getQueryTable(), context.getInsertOption().auditAction, count, auditType, auditUserComment);
+            if (!context.isCrossTypeImport() && !context.isCrossFolderImport()) // audit handled at table level
+            {
+                AuditBehaviorType auditType = (AuditBehaviorType) context.getConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior);
+                String auditUserComment = (String) context.getConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment);
+                boolean skipAuditLevelCheck = false;
+                if (context.getConfigParameterBoolean(QueryUpdateService.ConfigParameters.BulkLoad))
+                {
+                    if (getQueryTable().getEffectiveAuditBehavior(auditType) == AuditBehaviorType.DETAILED) // allow ETL to demote audit level for bulkLoad
+                        skipAuditLevelCheck = true;
+                }
+                getQueryTable().getAuditHandler(auditType).addSummaryAuditEvent(user, container, getQueryTable(), context.getInsertOption().auditAction, count, auditType, auditUserComment, skipAuditLevelCheck);
+            }
             return count;
         }
     }

api/src/org/labkey/api/query/FilteredTable.java

@@ -126,7 +126,10 @@ public FilteredTable(@NotNull TableInfo table, @NotNull SchemaType userSchema, @
         _importTemplates = _rootTable.getRawImportTemplates();
         _buttonBarConfig = BUTTON_BAR_NOT_SET;        // lazy copy button bar when asked
         if (_rootTable.supportsAuditTracking())
-            _auditBehaviorType = _rootTable.getAuditBehavior();
+        {
+            _auditBehaviorType = _rootTable.getDefaultAuditBehavior();
+            _xmlAuditBehaviorType = _rootTable.getXmlAuditBehaviorType();
+        }
 
         // We used to copy the titleColumn from table, but this forced all ColumnInfos to load.  Now, delegate
         // to _rootTable lazily, allowing overrides.