From 45b888250f45e82e36bd52b8a161d3556266f33a Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Mon, 29 Sep 2025 23:14:08 -0700 Subject: [PATCH 01/11] Initial support for moving list rows --- .../api/audit/AbstractAuditTypeProvider.java | 7 +- .../org/labkey/api/data/ContainerManager.java | 6261 +++++++++-------- .../labkey/api/exp/list/ListDefinition.java | 4 +- .../org/labkey/api/exp/list/ListService.java | 130 +- .../org/labkey/api/query/QueryService.java | 2 +- .../experiment/api/ExperimentServiceImpl.java | 4 +- .../labkey/list/model/ListAuditProvider.java | 9 +- .../labkey/list/model/ListDefinitionImpl.java | 26 +- .../org/labkey/list/model/ListManager.java | 4 - .../list/model/ListQueryUpdateService.java | 1453 ++-- .../org/labkey/query/QueryServiceImpl.java | 2 +- .../query/audit/QueryUpdateAuditProvider.java | 622 +- 12 files changed, 4387 insertions(+), 4137 deletions(-) diff --git a/api/src/org/labkey/api/audit/AbstractAuditTypeProvider.java b/api/src/org/labkey/api/audit/AbstractAuditTypeProvider.java index ff5fdfdc722..ccab9d16d9b 100644 --- a/api/src/org/labkey/api/audit/AbstractAuditTypeProvider.java +++ b/api/src/org/labkey/api/audit/AbstractAuditTypeProvider.java @@ -395,11 +395,6 @@ else if (value instanceof Date date) public int moveEvents(Container targetContainer, String idColumnName, Collection ids) { - TableInfo auditTable = createStorageTableInfo(); - SQLFragment sql = new SQLFragment("UPDATE ").append(auditTable) - .append(" SET container = ").appendValue(targetContainer) - .append(" WHERE ").append(idColumnName); - auditTable.getSchema().getSqlDialect().appendInClauseSql(sql, ids); - return new SqlExecutor(auditTable.getSchema()).execute(sql); + return ContainerManager.updateContainer(createStorageTableInfo(), idColumnName, ids, targetContainer, null, false); } } diff --git a/api/src/org/labkey/api/data/ContainerManager.java b/api/src/org/labkey/api/data/ContainerManager.java index ecae4509833..415810daa51 100644 --- a/api/src/org/labkey/api/data/ContainerManager.java +++ b/api/src/org/labkey/api/data/ContainerManager.java @@ -1,3124 +1,3137 @@ -/* - * Copyright (c) 2005-2018 Fred Hutchinson Cancer Research Center - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.data; - -import com.google.common.base.Enums; -import org.apache.commons.collections4.MultiValuedMap; -import org.apache.commons.collections4.multimap.ArrayListValuedHashMap; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.xmlbeans.XmlObject; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.labkey.api.Constants; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.admin.FolderExportContext; -import org.labkey.api.admin.FolderImportContext; -import org.labkey.api.admin.FolderImporterImpl; -import org.labkey.api.admin.FolderWriterImpl; -import org.labkey.api.admin.StaticLoggerGetter; -import org.labkey.api.attachments.AttachmentParent; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.provider.ContainerAuditProvider; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.ConcurrentHashSet; -import org.labkey.api.collections.IntHashMap; -import org.labkey.api.data.Container.ContainerException; -import org.labkey.api.data.Container.LockState; -import org.labkey.api.data.PropertyManager.WritablePropertyMap; -import org.labkey.api.data.SimpleFilter.InClause; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.data.validator.ColumnValidators; -import org.labkey.api.event.PropertyChange; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.module.FolderType; -import org.labkey.api.module.FolderTypeManager; -import org.labkey.api.module.Module; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.portal.ProjectUrls; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.SimpleValidationError; -import org.labkey.api.query.ValidationException; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.Group; -import org.labkey.api.security.MutableSecurityPolicy; -import org.labkey.api.security.SecurityLogger; -import org.labkey.api.security.SecurityManager; -import org.labkey.api.security.SecurityPolicyManager; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.CreateProjectPermission; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.InsertPermission; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.security.roles.AuthorRole; -import org.labkey.api.security.roles.ReaderRole; -import org.labkey.api.security.roles.Role; -import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.settings.AppProps; -import org.labkey.api.test.TestTimeout; -import org.labkey.api.test.TestWhen; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JunitUtil; -import org.labkey.api.util.MinorConfigurationException; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Path; -import org.labkey.api.util.QuietCloser; -import org.labkey.api.util.ReentrantLockWithName; -import org.labkey.api.util.ResultSetUtil; -import org.labkey.api.util.TestContext; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.FolderTab; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.NavTreeManager; -import org.labkey.api.view.Portal; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.view.ViewContext; -import org.labkey.api.writer.MemoryVirtualFile; -import org.labkey.folder.xml.FolderDocument; -import org.labkey.remoteapi.collections.CaseInsensitiveHashMap; -import org.springframework.validation.BindException; -import org.springframework.validation.Errors; - -import java.beans.PropertyChangeEvent; -import java.beans.PropertyChangeListener; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.ListIterator; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -import static org.labkey.api.action.SpringActionController.ERROR_GENERIC; - -/** - * This class manages a hierarchy of collections, backed by a database table called Containers. - * Containers are named using filesystem-like paths e.g. /proteomics/comet/. Each path - * maps to a UID and set of permissions. The current security scheme allows ACLs - * to be specified explicitly on the directory or completely inherited. ACLs are not combined. - *

- * NOTE: we act like java.io.File(). Paths start with forward-slash, but do not end with forward-slash. - * The root container's name is '/'. This means that it is not always the case that - * me.getPath() == me.getParent().getPath() + "/" + me.getName() - *

- * The synchronization goals are to keep invalid containers from creeping into the cache. For example, once - * a container is deleted, it should never get put back in the cache. We accomplish this by synchronizing on - * the removal from the cache, and the database lookup/cache insertion. While a container is in the middle - * of being deleted, it's OK for other clients to see it because FKs enforce that it's always internally - * consistent, even if some of the data has already been deleted. - */ -public class ContainerManager -{ - private static final Logger LOG = LogHelper.getLogger(ContainerManager.class, "Container (projects, folders, and workbooks) retrieval and management"); - private static final CoreSchema CORE = CoreSchema.getInstance(); - - private static final String PROJECT_LIST_ID = "Projects"; - - public static final String HOME_PROJECT_PATH = "/home"; - public static final String DEFAULT_SUPPORT_PROJECT_PATH = HOME_PROJECT_PATH + "/support"; - - private static final Cache CACHE_PATH = CacheManager.getCache(Constants.getMaxContainers(), CacheManager.DAY, "Containers by Path"); - private static final Cache CACHE_ENTITY_ID = CacheManager.getCache(Constants.getMaxContainers(), CacheManager.DAY, "Containers by EntityId"); - private static final Cache> CACHE_CHILDREN = CacheManager.getCache(Constants.getMaxContainers(), CacheManager.DAY, "Child EntityIds of Containers"); - private static final ReentrantLock DATABASE_QUERY_LOCK = new ReentrantLockWithName(ContainerManager.class, "DATABASE_QUERY_LOCK"); - public static final String FOLDER_TYPE_PROPERTY_SET_NAME = "folderType"; - public static final String FOLDER_TYPE_PROPERTY_NAME = "name"; - public static final String FOLDER_TYPE_PROPERTY_TABTYPE_OVERRIDDEN = "ctFolderTypeOverridden"; - public static final String TABFOLDER_CHILDREN_DELETED = "tabChildrenDeleted"; - public static final String AUDIT_SETTINGS_PROPERTY_SET_NAME = "containerAuditSettings"; - public static final String REQUIRE_USER_COMMENTS_PROPERTY_NAME = "requireUserComments"; - - private static final List _resourceProviders = new CopyOnWriteArrayList<>(); - - // containers that are being constructed, used to suppress events before fireCreateContainer() - private static final Set _constructing = new ConcurrentHashSet<>(); - - - /** enum of properties you can see in property change events */ - public enum Property - { - Name, - Parent, - Policy, - /** The default or active set of modules in the container has changed */ - Modules, - FolderType, - WebRoot, - AttachmentDirectory, - PipelineRoot, - Title, - Description, - SiteRoot, - StudyChange, - EndpointDirectory, - CloudStores - } - - static Path makePath(Container parent, String name) - { - if (null == parent) - return new Path(name); - return parent.getParsedPath().append(name, true); - } - - public static Container createMockContainer() - { - return new Container(null, "MockContainer", "01234567-8901-2345-6789-012345678901", 99999999, 0, new Date(), User.guest.getUserId(), true); - } - - private static Container createRoot() - { - Map m = new HashMap<>(); - m.put("Parent", null); - m.put("Name", ""); - Table.insert(null, CORE.getTableInfoContainers(), m); - - return getRoot(); - } - - private static DbScope.Transaction ensureTransaction() - { - return CORE.getSchema().getScope().ensureTransaction(DATABASE_QUERY_LOCK); - } - - private static int getNewChildSortOrder(Container parent) - { - int nextSortOrderVal = 0; - - List children = parent.getChildren(); - if (children != null) - { - for (Container child : children) - { - // find the max sort order value for the set of children - nextSortOrderVal = Math.max(nextSortOrderVal, child.getSortOrder()); - } - } - - // custom sorting applies: put new container at the end. - if (nextSortOrderVal > 0) - return nextSortOrderVal + 1; - - // we're sorted alphabetically - return 0; - } - - // TODO: Make private and force callers to use ensureContainer instead? - // TODO: Handle root creation here? - @NotNull - public static Container createContainer(Container parent, String name, @NotNull User user) - { - return createContainer(parent, name, null, null, NormalContainerType.NAME, user, null, null); - } - - public static final String WORKBOOK_DBSEQUENCE_NAME = "org.labkey.api.data.Workbooks"; - - // TODO: Pass in FolderType (separate from the container type of workbook, etc) and transact it with container creation? - @NotNull - public static Container createContainer(Container parent, String name, @Nullable String title, @Nullable String description, String type, @NotNull User user) - { - return createContainer(parent, name, title, description, type, user, null, null); - } - - @NotNull - public static Container createContainer(Container parent, String name, @Nullable String title, @Nullable String description, String type, @NotNull User user, @Nullable String auditMsg) - { - return createContainer(parent, name, title, description, type, user, auditMsg, null); - } - - @NotNull - public static Container createContainer(Container parent, String name, @Nullable String title, @Nullable String description, String type, @NotNull User user, @Nullable String auditMsg, - Consumer configureContainer) - { - ContainerType cType = ContainerTypeRegistry.get().getType(type); - if (cType == null) - throw new IllegalArgumentException("Unknown container type: " + type); - - // TODO: move this to ContainerType? - long sortOrder; - if (cType instanceof WorkbookContainerType) - { - sortOrder = DbSequenceManager.get(parent, WORKBOOK_DBSEQUENCE_NAME).next(); - - // Default workbook names are simply "" - if (name == null) - name = String.valueOf(sortOrder); - } - else - { - sortOrder = getNewChildSortOrder(parent); - } - - if (!parent.canHaveChildren()) - throw new IllegalArgumentException("Parent of a container must not be a " + parent.getContainerType().getName()); - - StringBuilder error = new StringBuilder(); - if (!Container.isLegalName(name, parent.isRoot(), error)) - throw new ApiUsageException(error.toString()); - - if (!Container.isLegalTitle(title, error)) - throw new ApiUsageException(error.toString()); - - Path path = makePath(parent, name); - SQLException sqlx = null; - Map insertMap = null; - - GUID entityId = new GUID(); - Container c; - - try - { - _constructing.add(entityId); - - try - { - Map m = new CaseInsensitiveHashMap<>(); - m.put("Parent", parent.getId()); - m.put("Name", name); - m.put("Title", title); - m.put("SortOrder", sortOrder); - m.put("EntityId", entityId); - if (null != description) - m.put("Description", description); - m.put("Type", type); - insertMap = Table.insert(user, CORE.getTableInfoContainers(), m); - } - catch (RuntimeSQLException x) - { - if (!x.isConstraintException()) - throw x; - sqlx = x.getSQLException(); - } - - _clearChildrenFromCache(parent); - - c = insertMap == null ? null : getForId(entityId); - - if (null == c) - { - if (null != sqlx) - throw new RuntimeSQLException(sqlx); - else - throw new RuntimeException("Container for path '" + path + "' was not created properly."); - } - - User savePolicyUser = user; - if (c.isProject() && !c.hasPermission(user, AdminPermission.class) && ContainerManager.getRoot().hasPermission(user, CreateProjectPermission.class)) - { - // Special case for project creators who don't necessarily yet have permission to save the policy of - // the project they just created - savePolicyUser = User.getAdminServiceUser(); - } - - // Workbooks inherit perms from their parent so don't create a policy if this is a workbook - if (c.isContainerFor(ContainerType.DataType.permissions)) - { - SecurityManager.setAdminOnlyPermissions(c, savePolicyUser); - } - - _removeFromCache(c, true); // seems odd, but it removes c.getProject() which clears other things from the cache - - // Initialize the list of active modules in the Container - c.getActiveModules(true, true, user); - - if (c.isProject()) - { - SecurityManager.createNewProjectGroups(c, savePolicyUser); - } - else - { - // If current user does NOT have admin permission on this container or the project has been - // explicitly set to have new subfolders inherit permissions, then inherit permissions - // (otherwise they would not be able to see the folder) - boolean hasAdminPermission = c.hasPermission(user, AdminPermission.class); - if ((!hasAdminPermission && !user.hasRootAdminPermission()) || SecurityManager.shouldNewSubfoldersInheritPermissions(c.getProject())) - SecurityManager.setInheritPermissions(c); - } - - // NOTE parent caches some info about children (e.g. hasWorkbookChildren) - // since mutating cached objects is frowned upon, just uncache parent - // CONSIDER: we could perhaps only uncache if the child is a workbook, but I think this reasonable - _removeFromCache(parent, true); - - if (null != configureContainer) - configureContainer.accept(c); - } - finally - { - _constructing.remove(entityId); - } - - fireCreateContainer(c, user, auditMsg); - - return c; - } - - public static void addSecurableResourceProvider(ContainerSecurableResourceProvider provider) - { - _resourceProviders.add(provider); - } - - public static List getSecurableResourceProviders() - { - return Collections.unmodifiableList(_resourceProviders); - } - - public static Container createContainerFromTemplate(Container parent, String name, String title, Container templateContainer, User user, FolderExportContext exportCtx, Consumer afterCreateHandler) throws Exception - { - MemoryVirtualFile vf = new MemoryVirtualFile(); - - // export objects from the source template folder - FolderWriterImpl writer = new FolderWriterImpl(); - writer.write(templateContainer, exportCtx, vf); - - // create the new target container - Container c = createContainer(parent, name, title, null, NormalContainerType.NAME, user, null, afterCreateHandler); - - // import objects into the target folder - XmlObject folderXml = vf.getXmlBean("folder.xml"); - if (folderXml instanceof FolderDocument folderDoc) - { - FolderImportContext importCtx = new FolderImportContext(user, c, folderDoc, null, new StaticLoggerGetter(LogManager.getLogger(FolderImporterImpl.class)), vf); - - FolderImporterImpl importer = new FolderImporterImpl(); - importer.process(null, importCtx, vf); - } - - return c; - } - - public static void setRequireAuditComments(Container container, User user, @NotNull Boolean required) - { - WritablePropertyMap props = PropertyManager.getWritableProperties(container, AUDIT_SETTINGS_PROPERTY_SET_NAME, true); - String originalValue = props.get(REQUIRE_USER_COMMENTS_PROPERTY_NAME); - props.put(REQUIRE_USER_COMMENTS_PROPERTY_NAME, required.toString()); - props.save(); - - addAuditEvent(user, container, - "Changed " + REQUIRE_USER_COMMENTS_PROPERTY_NAME + " from \"" + - originalValue + "\" to \"" + required + "\""); - } - - public static void setFolderType(Container c, FolderType folderType, User user, BindException errors) - { - FolderType oldType = c.getFolderType(); - - if (folderType.equals(oldType)) - return; - - List errorStrings = new ArrayList<>(); - - if (!c.isProject() && folderType.isProjectOnlyType()) - errorStrings.add("Cannot set a subfolder to " + folderType.getName() + " because it is a project-only folder type."); - - // Check for any containers that need to be moved into container tabs - if (errorStrings.isEmpty() && folderType.hasContainerTabs()) - { - List childTabFoldersNonMatchingTypes = new ArrayList<>(); - List containersBecomingTabs = findAndCheckContainersMatchingTabs(c, folderType, childTabFoldersNonMatchingTypes, errorStrings); - - if (errorStrings.isEmpty()) - { - if (!containersBecomingTabs.isEmpty()) - { - // Make containers tab container; Folder tab will find them by name - try (DbScope.Transaction transaction = ensureTransaction()) - { - for (Container container : containersBecomingTabs) - updateType(container, TabContainerType.NAME, user); - - transaction.commit(); - } - } - - // Check these and change type unless they were overridden explicitly - for (Container container : childTabFoldersNonMatchingTypes) - { - if (!isContainerTabTypeOverridden(container)) - { - FolderTab newTab = folderType.findTab(container.getName()); - assert null != newTab; // There must be a tab because it caused the container to get into childTabFoldersNonMatchingTypes - FolderType newType = newTab.getFolderType(); - if (null == newType) - newType = FolderType.NONE; // default to NONE - setFolderType(container, newType, user, errors); - } - } - } - } - - if (errorStrings.isEmpty()) - { - oldType.unconfigureContainer(c, user); - WritablePropertyMap props = PropertyManager.getWritableProperties(c, FOLDER_TYPE_PROPERTY_SET_NAME, true); - props.put(FOLDER_TYPE_PROPERTY_NAME, folderType.getName()); - - if (c.isContainerTab()) - { - boolean containerTabTypeOverridden = false; - FolderTab tab = c.getParent().getFolderType().findTab(c.getName()); - if (null != tab && !folderType.equals(tab.getFolderType())) - containerTabTypeOverridden = true; - props.put(FOLDER_TYPE_PROPERTY_TABTYPE_OVERRIDDEN, Boolean.toString(containerTabTypeOverridden)); - } - props.save(); - - notifyContainerChange(c.getId(), Property.FolderType, user); - folderType.configureContainer(c, user); // Configure new only after folder type has been changed - - // TODO: Not needed? I don't think we've changed the container's state. - _removeFromCache(c, false); - } - else - { - for (String errorString : errorStrings) - errors.reject(SpringActionController.ERROR_MSG, errorString); - } - } - - public static void checkContainerValidity(Container c) throws ContainerException - { - // Check container for validity; in rare cases user may have changed their custom folderType.xml and caused - // duplicate subfolders (same name) to exist - // Get list of child containers that are not container tabs, but match container tabs; these are bad - FolderType folderType = getFolderType(c); - List errorStrings = new ArrayList<>(); - List childTabFoldersNonMatchingTypes = new ArrayList<>(); - List containersMatchingTabs = findAndCheckContainersMatchingTabs(c, folderType, childTabFoldersNonMatchingTypes, errorStrings); - if (!containersMatchingTabs.isEmpty()) - { - throw new Container.ContainerException("Folder " + c.getPath() + - " has a subfolder with the same name as a container tab folder, which is an invalid state." + - " This may have been caused by changing the folder type's tabs after this folder was set to its folder type." + - " An administrator should either delete the offending subfolder or change the folder's folder type.\n"); - } - } - - public static List findAndCheckContainersMatchingTabs(Container c, FolderType folderType, - List childTabFoldersNonMatchingTypes, List errorStrings) - { - List containersMatchingTabs = new ArrayList<>(); - for (FolderTab folderTab : folderType.getDefaultTabs()) - { - if (folderTab.getTabType() == FolderTab.TAB_TYPE.Container) - { - for (Container child : c.getChildren()) - { - if (child.getName().equalsIgnoreCase(folderTab.getName())) - { - if (!child.getFolderType().getName().equalsIgnoreCase(folderTab.getFolderTypeName())) - { - if (child.isContainerTab()) - childTabFoldersNonMatchingTypes.add(child); // Tab type doesn't match child tab folder - else - errorStrings.add("Child folder " + child.getName() + - " matches container tab, but folder type " + child.getFolderType().getName() + " doesn't match tab's folder type " + - folderTab.getFolderTypeName() + "."); - } - - int childCount = child.getChildren().size(); - if (childCount > 0) - { - errorStrings.add("Child folder " + child.getName() + - " matches container tab, but cannot be converted to a tab folder because it has " + childCount + " children."); - } - - if (!child.isConvertibleToTab()) - { - errorStrings.add("Child folder " + child.getName() + - " matches container tab, but cannot be converted to a tab folder because it is a " + child.getContainerNoun() + "."); - } - - if (!child.isContainerTab()) - containersMatchingTabs.add(child); - - break; // we found name match; can't be another - } - } - } - } - return containersMatchingTabs; - } - - private static final Set containersWithBadFolderTypes = new ConcurrentHashSet<>(); - - @NotNull - public static FolderType getFolderType(Container c) - { - String name = getFolderTypeName(c); - FolderType folderType; - - if (null != name) - { - folderType = FolderTypeManager.get().getFolderType(name); - - if (null == folderType) - { - // If we're upgrading then folder types won't be defined yet... don't warn in that case. - if (!ModuleLoader.getInstance().isUpgradeInProgress() && - !ModuleLoader.getInstance().isUpgradeRequired() && - !containersWithBadFolderTypes.contains(c)) - { - LOG.warn("No such folder type " + name + " for folder " + c.toString()); - containersWithBadFolderTypes.add(c); - } - - folderType = FolderType.NONE; - } - } - else - folderType = FolderType.NONE; - - return folderType; - } - - /** - * Most code should call getFolderType() instead. - * Useful for finding the name of the folder type BEFORE startup is complete, so the FolderType itself - * may not be available. - */ - @Nullable - public static String getFolderTypeName(Container c) - { - Map props = PropertyManager.getProperties(c, FOLDER_TYPE_PROPERTY_SET_NAME); - return props.get(FOLDER_TYPE_PROPERTY_NAME); - } - - - @NotNull - public static Map getFolderTypeNameContainerCounts(Container root) - { - Map nameCounts = new TreeMap<>(); - for (Container c : getAllChildren(root)) - { - Integer count = nameCounts.get(c.getFolderType().getName()); - if (null == count) - { - count = Integer.valueOf(0); - } - nameCounts.put(c.getFolderType().getName(), ++count); - } - return nameCounts; - } - - @NotNull - public static Map getProductFoldersMetrics(@NotNull FolderType folderType) - { - Container root = getRoot(); - Map metrics = new TreeMap<>(); - List counts = new ArrayList<>(); - for (Container c : root.getChildren()) - { - if (!c.getFolderType().getName().equals(folderType.getName())) - continue; - - int childCount = c.getChildren().stream().filter(Container::isInFolderNav).toList().size(); - counts.add(childCount); - } - - int totalFolderTypeMatch = counts.size(); - if (totalFolderTypeMatch == 0) - return metrics; - - Collections.sort(counts); - int median = counts.get((totalFolderTypeMatch - 1)/2); - if (totalFolderTypeMatch % 2 == 0 ) - { - int low = counts.get(totalFolderTypeMatch/2 - 1); - int high = counts.get(totalFolderTypeMatch/2); - median = Math.round((low + high) / 2.0f); - } - int maxProjectsCount = counts.get(totalFolderTypeMatch - 1); - int totalProjectsCount = counts.stream().mapToInt(Integer::intValue).sum(); - int averageProjectsCount = Math.round((float) totalProjectsCount /totalFolderTypeMatch); - - metrics.put("totalSubProjectsCount", totalProjectsCount); - metrics.put("averageSubProjectsPerHomeProject", averageProjectsCount); - metrics.put("medianSubProjectsCountPerHomeProject", median); - metrics.put("maxSubProjectsCountInHomeProject", maxProjectsCount); - - return metrics; - } - - public static boolean isContainerTabTypeThisOrChildrenOverridden(Container c) - { - if (isContainerTabTypeOverridden(c)) - return true; - if (c.getFolderType().hasContainerTabs()) - { - for (Container child : c.getChildren()) - { - if (child.isContainerTab() && isContainerTabTypeOverridden(child)) - return true; - } - } - return false; - } - - public static boolean isContainerTabTypeOverridden(Container c) - { - Map props = PropertyManager.getProperties(c, FOLDER_TYPE_PROPERTY_SET_NAME); - String overridden = props.get(FOLDER_TYPE_PROPERTY_TABTYPE_OVERRIDDEN); - return (null != overridden) && overridden.equalsIgnoreCase("true"); - } - - private static void setContainerTabDeleted(Container c, String tabName, String folderTypeName) - { - // Add prop in this category - WritablePropertyMap props = PropertyManager.getWritableProperties(c, TABFOLDER_CHILDREN_DELETED, true); - props.put(getDeletedTabKey(tabName, folderTypeName), "true"); - props.save(); - } - - public static void clearContainerTabDeleted(Container c, String tabName, String folderTypeName) - { - WritablePropertyMap props = PropertyManager.getWritableProperties(c, TABFOLDER_CHILDREN_DELETED, true); - String key = getDeletedTabKey(tabName, folderTypeName); - if (props.containsKey(key)) - { - props.remove(key); - props.save(); - } - } - - public static boolean hasContainerTabBeenDeleted(Container c, String tabName, String folderTypeName) - { - // We keep arbitrary number of deleted children tabs using suffix 0, 1, 2.... - Map props = PropertyManager.getProperties(c, TABFOLDER_CHILDREN_DELETED); - return props.containsKey(getDeletedTabKey(tabName, folderTypeName)); - } - - private static String getDeletedTabKey(String tabName, String folderTypeName) - { - return tabName + "-TABDELETED-FOLDER-" + folderTypeName; - } - - @NotNull - public static Container ensureContainer(@NotNull String path, @NotNull User user) - { - return ensureContainer(Path.parse(path), user); - } - - @NotNull - public static Container ensureContainer(@NotNull Path path, @NotNull User user) - { - Container c = null; - - try - { - c = getForPath(path); - } - catch (RootContainerException e) - { - // Ignore this -- root doesn't exist yet - } - - if (null == c) - { - if (path.isEmpty()) - c = createRoot(); - else - { - Path parentPath = path.getParent(); - c = ensureContainer(parentPath, user); - c = createContainer(c, path.getName(), null, null, NormalContainerType.NAME, user); - } - } - return c; - } - - - @NotNull - public static Container ensureContainer(Container parent, String name, User user) - { - // NOTE: Running outside a tx doesn't seem to be necessary. -// if (CORE.getSchema().getScope().isTransactionActive()) -// throw new IllegalStateException("Transaction should not be active"); - - Container c = null; - - try - { - c = getForPath(makePath(parent,name)); - } - catch (RootContainerException e) - { - // Ignore this -- root doesn't exist yet - } - - if (null == c) - { - c = createContainer(parent, name, user); - } - return c; - } - - public static void updateDescription(Container container, String description, User user) - throws ValidationException - { - ColumnValidators.validate(CORE.getTableInfoContainers().getColumn("Title"), null, 1, description); - - //For some reason, there is no primary key defined on core.containers - //so we can't use Table.update here - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - sql.append(" SET Description=? WHERE RowID=?").add(description).add(container.getRowId()); - new SqlExecutor(CORE.getSchema()).execute(sql); - - String oldValue = container.getDescription(); - _removeFromCache(container, false); - container = getForRowId(container.getRowId()); - ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(container, user, Property.Description, oldValue, description); - firePropertyChangeEvent(evt); - } - - public static void updateSearchable(Container container, boolean searchable, User user) - { - //For some reason, there is no primary key defined on core.containers - //so we can't use Table.update here - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - sql.append(" SET Searchable=? WHERE RowID=?").add(searchable).add(container.getRowId()); - new SqlExecutor(CORE.getSchema()).execute(sql); - - _removeFromCache(container, false); - } - - public static void updateLockState(Container container, LockState lockState, @NotNull Runnable auditRunnable) - { - //For some reason there is no primary key defined on core.containers - //so we can't use Table.update here - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - sql.append(" SET LockState = ?, ExpirationDate = NULL WHERE RowID = ?").add(lockState).add(container.getRowId()); - new SqlExecutor(CORE.getSchema()).execute(sql); - - _removeFromCache(container, false); - - auditRunnable.run(); - } - - public static List getExcludedProjects() - { - return getProjects().stream() - .filter(p->p.getLockState() == Container.LockState.Excluded) - .collect(Collectors.toList()); - } - - public static List getNonExcludedProjects() - { - return getProjects().stream() - .filter(p->p.getLockState() != Container.LockState.Excluded) - .collect(Collectors.toList()); - } - - public static void setExcludedProjects(Collection ids, @NotNull Runnable auditRunnable) - { - // First clear all existing "Excluded" states - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - sql.append(" SET LockState = NULL, ExpirationDate = NULL WHERE LockState = ?").add(LockState.Excluded); - new SqlExecutor(CORE.getSchema()).execute(sql); - - // Now set the passed-in projects to "Excluded" - if (!ids.isEmpty()) - { - ColumnInfo entityIdCol = CORE.getTableInfoContainers().getColumn("EntityId"); - Filter inClauseFilter = new SimpleFilter(new InClause(entityIdCol.getFieldKey(), ids)); - SQLFragment frag = new SQLFragment("UPDATE "); - frag.append(CORE.getTableInfoContainers().getSelectName()); - frag.append(" SET LockState = ?, ExpirationDate = NULL "); - frag.add(LockState.Excluded); - frag.append(inClauseFilter.getSQLFragment(CORE.getSqlDialect(), "c", Map.of(entityIdCol.getFieldKey(), entityIdCol))); - new SqlExecutor(CORE.getSchema()).execute(frag); - } - - clearCache(); - - auditRunnable.run(); - } - - public static void archiveContainer(User user, Container container, boolean archive) - { - if (container.isRoot() || container.isProject() || container.isAppHomeFolder()) - throw new ApiUsageException("Archive action not supported for this folder."); - - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers().getSelectName()); - if (archive) - { - sql.append(" SET LockState = ? "); - sql.add(LockState.Archived); - sql.append(" WHERE LockState IS NULL "); - } - else - { - sql.append(" SET LockState = NULL WHERE LockState = ? "); - sql.add(LockState.Archived); - } - sql.append("AND EntityId = ? "); - sql.add(container.getEntityId()); - new SqlExecutor(CORE.getSchema()).execute(sql); - - clearCache(); - - addAuditEvent(user, container, archive ? "Container has been archived." : "Archived container has been restored."); - } - - public static void updateExpirationDate(Container container, LocalDate expirationDate, @NotNull Runnable auditRunnable) - { - //For some reason there is no primary key defined on core.containers - //so we can't use Table.update here - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - // Note: jTDS doesn't support LocalDate, so convert to java.sql.Date - sql.append(" SET ExpirationDate = ? WHERE RowID = ?").add(java.sql.Date.valueOf(expirationDate)).add(container.getRowId()); - - new SqlExecutor(CORE.getSchema()).execute(sql); - - _removeFromCache(container, false); - - auditRunnable.run(); - } - - public static void updateType(Container container, String newType, User user) - { - //For some reason there is no primary key defined on core.containers - //so we can't use Table.update here - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - sql.append(" SET Type=? WHERE RowID=?").add(newType).add(container.getRowId()); - new SqlExecutor(CORE.getSchema()).execute(sql); - - _removeFromCache(container, false); - } - - public static void updateTitle(Container container, String title, User user) - throws ValidationException - { - ColumnValidators.validate(CORE.getTableInfoContainers().getColumn("Title"), null, 1, title); - - //For some reason there is no primary key defined on core.containers - //so we can't use Table.update here - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - sql.append(" SET Title=? WHERE RowID=?").add(title).add(container.getRowId()); - new SqlExecutor(CORE.getSchema()).execute(sql); - - _removeFromCache(container, false); - String oldValue = container.getTitle(); - container = getForRowId(container.getRowId()); - ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(container, user, Property.Title, oldValue, title); - firePropertyChangeEvent(evt); - } - - public static void uncache(Container c) - { - _removeFromCache(c, true); - } - - public static final String SHARED_CONTAINER_PATH = "/Shared"; - - @NotNull - public static Container getSharedContainer() - { - return ensureContainer(Path.parse(SHARED_CONTAINER_PATH), User.getAdminServiceUser()); - } - - public static List getChildren(Container parent) - { - return new ArrayList<>(getChildrenMap(parent).values()); - } - - // Default is to include all types of children, as seems only appropriate - public static List getChildren(Container parent, User u, Class perm) - { - return getChildren(parent, u, perm, null, ContainerTypeRegistry.get().getTypeNames()); - } - - public static List getChildren(Container parent, User u, Class perm, Set roles) - { - return getChildren(parent, u, perm, roles, ContainerTypeRegistry.get().getTypeNames()); - } - - public static List getChildren(Container parent, User u, Class perm, String typeIncluded) - { - return getChildren(parent, u, perm, null, Collections.singleton(typeIncluded)); - } - - public static List getChildren(Container parent, User u, Class perm, Set roles, Set includedTypes) - { - List children = new ArrayList<>(); - for (Container child : getChildrenMap(parent).values()) - if (includedTypes.contains(child.getContainerType().getName()) && child.hasPermission(u, perm, roles)) - children.add(child); - - return children; - } - - public static List getAllChildren(Container parent, User u) - { - return getAllChildren(parent, u, ReadPermission.class, null, ContainerTypeRegistry.get().getTypeNames()); - } - - public static List getAllChildren(Container parent, User u, Class perm) - { - return getAllChildren(parent, u, perm, null, ContainerTypeRegistry.get().getTypeNames()); - } - - // Default is to include all types of children - public static List getAllChildren(Container parent, User u, Class perm, Set roles) - { - return getAllChildren(parent, u, perm, roles, ContainerTypeRegistry.get().getTypeNames()); - } - - public static List getAllChildren(Container parent, User u, Class perm, String typeIncluded) - { - return getAllChildren(parent, u, perm, null, Collections.singleton(typeIncluded)); - } - - public static List getAllChildren(Container parent, User u, Class perm, Set roles, Set typesIncluded) - { - Set allChildren = getAllChildren(parent); - List result = new ArrayList<>(allChildren.size()); - - for (Container container : allChildren) - { - if (typesIncluded.contains(container.getContainerType().getName()) && container.hasPermission(u, perm, roles)) - { - result.add(container); - } - } - - return result; - } - - // Returns the next available child container name based on the baseName - public static String getAvailableChildContainerName(Container c, String baseName) - { - List children = getChildren(c); - Map folders = new HashMap<>(children.size() * 2); - for (Container child : children) - folders.put(child.getName(), child); - - String availableContainerName = baseName; - int i = 1; - while (folders.containsKey(availableContainerName)) - { - availableContainerName = baseName + " " + i++; - } - - return availableContainerName; - } - - // Returns true only if user has the specified permission in the entire container tree starting at root - public static boolean hasTreePermission(Container root, User u, Class perm) - { - for (Container c : getAllChildren(root)) - if (!c.hasPermission(u, perm)) - return false; - - return true; - } - - private static Map getChildrenMap(Container parent) - { - if (!parent.canHaveChildren()) - { - // Optimization to avoid database query (important because some installs have tens of thousands of - // workbooks) when the container is a workbook, which is not allowed to have children - return Collections.emptyMap(); - } - - List childIds = CACHE_CHILDREN.get(parent.getEntityId()); - if (null == childIds) - { - try (DbScope.Transaction t = ensureTransaction()) - { - List children = new SqlSelector(CORE.getSchema(), - "SELECT * FROM " + CORE.getTableInfoContainers() + " WHERE Parent = ? ORDER BY SortOrder, LOWER(Name)", - parent.getId()).getArrayList(Container.class); - - childIds = new ArrayList<>(children.size()); - for (Container c : children) - { - childIds.add(c.getEntityId()); - _addToCache(c); - } - childIds = Collections.unmodifiableList(childIds); - CACHE_CHILDREN.put(parent.getEntityId(), childIds); - // No database changes to commit, but need to decrement the transaction counter - t.commit(); - } - } - - if (childIds.isEmpty()) - return Collections.emptyMap(); - - // Use a LinkedHashMap to preserve the order defined by the user - they're not necessarily alphabetical - Map ret = new LinkedHashMap<>(); - for (GUID id : childIds) - { - Container c = getForId(id); - if (null != c) - ret.put(c.getName(), c); - } - return Collections.unmodifiableMap(ret); - } - - public static Container getForRowId(int id) - { - Selector selector = new SqlSelector(CORE.getSchema(), new SQLFragment("SELECT * FROM " + CORE.getTableInfoContainers() + " WHERE RowId = ?", id)); - return selector.getObject(Container.class); - } - - public static @Nullable Container getForId(@NotNull GUID guid) - { - return guid != null ? getForId(guid.toString()) : null; - } - - public static @Nullable Container getForId(@Nullable String id) - { - //if the input string is not a GUID, just return null, - //so that we don't get a SQLException when the database - //tries to convert it to a unique identifier. - if (!GUID.isGUID(id)) - return null; - - GUID guid = new GUID(id); - - Container d = CACHE_ENTITY_ID.get(guid); - if (null != d) - return d; - - try (DbScope.Transaction t = ensureTransaction()) - { - Container result = new SqlSelector( - CORE.getSchema(), - "SELECT * FROM " + CORE.getTableInfoContainers() + " WHERE EntityId = ?", - id).getObject(Container.class); - if (result != null) - { - result = _addToCache(result); - } - // No database changes to commit, but need to decrement the counter - t.commit(); - - return result; - } - } - - public static Container getChild(Container c, String name) - { - Path path = c.getParsedPath().append(name); - - Container d = _getFromCachePath(path); - if (null != d) - return d; - - Map map = getChildrenMap(c); - return map.get(name); - } - - - public static Container getForURL(@NotNull ActionURL url) - { - Container ret = getForPath(url.getExtraPath()); - if (ret == null) - ret = getForId(StringUtils.strip(url.getExtraPath(), "/")); - return ret; - } - - - public static Container getForPath(@NotNull String path) - { - if (GUID.isGUID(path)) - { - Container c = getForId(path); - if (c != null) - return c; - } - - Path p = Path.parse(path); - return getForPath(p); - } - - public static Container getForPath(Path path) - { - Container d = _getFromCachePath(path); - if (null != d) - return d; - - // Special case for ROOT -- we want to throw instead of returning null - if (path.equals(Path.rootPath)) - { - try (DbScope.Transaction t = ensureTransaction()) - { - TableInfo tinfo = CORE.getTableInfoContainers(); - - // Unusual, but possible -- if cache loader hits an exception it can end up caching null - if (null == tinfo) - throw new RootContainerException("Container table could not be retrieved from the cache"); - - // This might be called at bootstrap, before schemas have been created - if (tinfo.getTableType() == DatabaseTableType.NOT_IN_DB) - throw new RootContainerException("Container table has not been created"); - - Container result = new SqlSelector(CORE.getSchema(),"SELECT * FROM " + tinfo + " WHERE Parent IS NULL").getObject(Container.class); - - if (result == null) - throw new RootContainerException("Root container does not exist"); - - _addToCache(result); - // No database changes to commit, but need to decrement the counter - t.commit(); - return result; - } - } - else - { - Path parent = path.getParent(); - String name = path.getName(); - Container dirParent = getForPath(parent); - - if (null == dirParent) - return null; - - Map map = getChildrenMap(dirParent); - return map.get(name); - } - } - - public static class RootContainerException extends RuntimeException - { - private RootContainerException(String message, Throwable cause) - { - super(message, cause); - } - - private RootContainerException(String message) - { - super(message); - } - } - - public static Container getRoot() - { - try - { - return getForPath("/"); - } - catch (MinorConfigurationException e) - { - // If the server is misconfigured, rethrow so some callers don't swallow it and other callers don't end up - // reporting it to mothership, Issue 50843. - throw e; - } - catch (Exception e) - { - // Some callers catch and ignore this exception, e.g., early in the bootstrap process - throw new RootContainerException("Root container can't be retrieved", e); - } - } - - public static void saveAliasesForContainer(Container container, List aliases, User user) - { - Set originalAliases = new CaseInsensitiveHashSet(getAliasesForContainer(container)); - Set newAliases = new CaseInsensitiveHashSet(aliases); - - if (originalAliases.equals(newAliases)) - { - return; - } - - try (DbScope.Transaction transaction = ensureTransaction()) - { - // Delete all of the aliases for the current container, plus any of the aliases that might be associated - // with another container right now - SQLFragment deleteSQL = new SQLFragment(); - deleteSQL.append("DELETE FROM "); - deleteSQL.append(CORE.getTableInfoContainerAliases()); - deleteSQL.append(" WHERE ContainerRowId = ? "); - deleteSQL.add(container.getRowId()); - if (!aliases.isEmpty()) - { - deleteSQL.append(" OR Path IN ("); - String separator = ""; - for (String alias : aliases) - { - deleteSQL.append(separator); - separator = ", "; - deleteSQL.append("LOWER(?)"); - deleteSQL.add(alias); - } - deleteSQL.append(")"); - } - new SqlExecutor(CORE.getSchema()).execute(deleteSQL); - - // Store the alias as LOWER() so that we can query against it using the index - for (String alias : newAliases) - { - SQLFragment insertSQL = new SQLFragment(); - insertSQL.append("INSERT INTO "); - insertSQL.append(CORE.getTableInfoContainerAliases()); - insertSQL.append(" (Path, ContainerRowId) VALUES (LOWER(?), ?)"); - insertSQL.add(alias); - insertSQL.add(container.getRowId()); - new SqlExecutor(CORE.getSchema()).execute(insertSQL); - } - - addAuditEvent(user, container, - "Changed folder aliases from \"" + - StringUtils.join(originalAliases, ", ") + "\" to \"" + - StringUtils.join(newAliases, ", ") + "\""); - - transaction.commit(); - } - } - - // Abstract base class used for attaching system resources (favorite icons, logos, stylesheets, sso auth logos) to folders and projects - public static abstract class ContainerParent implements AttachmentParent - { - private final Container _c; - - protected ContainerParent(Container c) - { - _c = c; - } - - @Override - public String getEntityId() - { - return _c.getId(); - } - - @Override - public String getContainerId() - { - return _c.getId(); - } - - public Container getContainer() - { - return _c; - } - } - - public static Container getHomeContainer() - { - return getForPath(HOME_PROJECT_PATH); - } - - public static List getProjects() - { - return getChildren(getRoot()); - } - - public static NavTree getProjectList(ViewContext context, boolean includeChildren) - { - User user = context.getUser(); - Container currentProject = context.getContainer().getProject(); - String projectNavTreeId = PROJECT_LIST_ID; - if (currentProject != null) - projectNavTreeId += currentProject.getId(); - - NavTree navTree = (NavTree) NavTreeManager.getFromCache(projectNavTreeId, context); - if (null != navTree) - return navTree; - - NavTree list = new NavTree("Projects"); - List projects = getProjects(); - - for (Container project : projects) - { - boolean shouldDisplay = project.shouldDisplay(user) && project.hasPermission("getProjectList()", user, ReadPermission.class); - boolean includeCurrentProject = includeChildren && currentProject != null && currentProject.equals(project); - - if (shouldDisplay || includeCurrentProject) - { - ActionURL startURL = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(project); - - if (includeChildren) - list.addChild(getFolderListForUser(project, context)); - else if (project.equals(getHomeContainer())) - list.addChild(new NavTree("Home", startURL)); - else - list.addChild(project.getTitle(), startURL); - } - } - - list.setId(projectNavTreeId); - NavTreeManager.cacheTree(list, context.getUser()); - - return list; - } - - public static NavTree getFolderListForUser(final Container project, ViewContext viewContext) - { - final boolean isNavAccessOpen = AppProps.getInstance().isNavigationAccessOpen(); - final Container c = viewContext.getContainer(); - final String cacheKey = isNavAccessOpen ? project.getId() : c.getId(); - - NavTree tree = (NavTree) NavTreeManager.getFromCache(cacheKey, viewContext); - if (null != tree) - return tree; - - try - { - assert SecurityLogger.indent("getFolderListForUser()"); - - User user = viewContext.getUser(); - String projectId = project.getId(); - - List folders = new ArrayList<>(getAllChildren(project)); - - Collections.sort(folders); - - Set containersInTree = new HashSet<>(); - - Map m = new HashMap<>(); - Map permission = new HashMap<>(); - - for (Container f : folders) - { - if (!f.isInFolderNav()) - continue; - - boolean hasPolicyRead = f.hasPermission(user, ReadPermission.class); - - boolean skip = ( - !hasPolicyRead || - !f.shouldDisplay(user) || - !f.hasPermission(user, ReadPermission.class) - ); - - //Always put the project and current container in... - if (skip && !f.equals(project) && !f.equals(c)) - continue; - - //HACK to make home link consistent... - String name = f.getTitle(); - if (name.equals("home") && f.equals(getHomeContainer())) - name = "Home"; - - NavTree t = new NavTree(name); - - // 34137: Support folder path expansion for containers where label != name - t.setId(f.getId()); - if (hasPolicyRead) - { - ActionURL url = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(f); - t.setHref(url.getEncodedLocalURIString()); - } - - boolean addFolder = false; - - if (isNavAccessOpen) - { - addFolder = true; - } - else - { - // 32718: If navigation access is not open then hide projects that aren't directly - // accessible in site folder navigation. - - if (f.equals(c) || f.isRoot() || (hasPolicyRead && f.isProject())) - { - // In current container, root, or readable project - addFolder = true; - } - else - { - boolean isAscendant = f.isDescendant(c); - boolean isDescendant = c.isDescendant(f); - boolean inActivePath = isAscendant || isDescendant; - boolean hasAncestryRead = false; - - if (inActivePath) - { - Container leaf = isAscendant ? f : c; - Container localRoot = isAscendant ? c : f; - - List ancestors = containersToRootList(leaf); - Collections.reverse(ancestors); - - for (Container p : ancestors) - { - if (!permission.containsKey(p.getId())) - permission.put(p.getId(), p.hasPermission(user, ReadPermission.class)); - boolean hasRead = permission.get(p.getId()); - - if (p.equals(localRoot)) - { - hasAncestryRead = hasRead; - break; - } - else if (!hasRead) - { - hasAncestryRead = false; - break; - } - } - } - else - { - hasAncestryRead = containersToRoot(f).stream().allMatch(p -> { - if (!permission.containsKey(p.getId())) - permission.put(p.getId(), p.hasPermission(user, ReadPermission.class)); - return permission.get(p.getId()); - }); - } - - if (hasPolicyRead && hasAncestryRead && inActivePath) - { - // Is in the direct readable lineage of the current container - addFolder = true; - } - else if (hasPolicyRead && f.getParent().equals(c.getParent())) - { - // Is a readable sibling of the current container - addFolder = true; - } - else if (hasAncestryRead) - { - // Is a part of a fully readable ancestry - addFolder = true; - } - } - - if (!addFolder) - LOG.debug("isNavAccessOpen restriction: \"" + f.getPath() + "\""); - } - - if (addFolder) - { - containersInTree.add(f); - m.put(f.getId(), t); - } - } - - //Ensure parents of any accessible folder are in the tree. If not add them with no link. - for (Container treeContainer : containersInTree) - { - if (!treeContainer.equals(project) && !containersInTree.contains(treeContainer.getParent())) - { - Set containersToRoot = containersToRoot(treeContainer); - //Possible will be added more than once, if several children are accessible, but that's OK... - for (Container missing : containersToRoot) - { - if (!m.containsKey(missing.getId())) - { - if (isNavAccessOpen) - { - NavTree noLinkTree = new NavTree(missing.getName()); - noLinkTree.setId(missing.getId()); - m.put(missing.getId(), noLinkTree); - } - else - { - if (!permission.containsKey(missing.getId())) - permission.put(missing.getId(), missing.hasPermission(user, ReadPermission.class)); - - if (!permission.get(missing.getId())) - { - NavTree noLinkTree = new NavTree(missing.getName()); - m.put(missing.getId(), noLinkTree); - } - else - { - NavTree linkTree = new NavTree(missing.getName()); - ActionURL url = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(missing); - linkTree.setHref(url.getEncodedLocalURIString()); - m.put(missing.getId(), linkTree); - } - } - } - } - } - } - - for (Container f : folders) - { - if (f.getId().equals(projectId)) - continue; - - NavTree child = m.get(f.getId()); - if (null == child) - continue; - - NavTree parent = m.get(f.getParent().getId()); - assert null != parent; //This should not happen anymore, we assure all parents are in tree. - if (null != parent) - parent.addChild(child); - } - - NavTree projectTree = m.get(projectId); - - projectTree.setId(cacheKey); - - NavTreeManager.cacheTree(projectTree, user); - return projectTree; - } - finally - { - assert SecurityLogger.outdent(); - } - } - - public static Set containersToRoot(Container child) - { - Set containersOnPath = new HashSet<>(); - Container current = child; - while (current != null && !current.isRoot()) - { - containersOnPath.add(current); - current = current.getParent(); - } - - return containersOnPath; - } - - /** - * Provides a sorted list of containers from the root to the child container provided. - * It does not include the root node. - * @param child Container from which the search is sourced. - * @return List sorted in order of distance from root. - */ - public static List containersToRootList(Container child) - { - List containers = new ArrayList<>(); - Container current = child; - while (current != null && !current.isRoot()) - { - containers.add(current); - current = current.getParent(); - } - - Collections.reverse(containers); - return containers; - } - - // Move a container to another part of the container tree. Careful: this method DOES NOT prevent you from orphaning - // an entire tree (e.g., by setting a container's parent to one of its children); the UI in AdminController does this. - // - // NOTE: Beware side-effect of changing ACLs and GROUPS if a container changes projects - // - // @return true if project has changed (should probably redirect to security page) - public static boolean move(Container c, final Container newParent, User user) throws ValidationException - { - if (!isRenameable(c)) - { - throw new IllegalArgumentException("Can't move container " + c.getPath()); - } - - try (QuietCloser ignored = lockForMutation(MutatingOperation.move, c)) - { - List errors = new ArrayList<>(); - for (ContainerListener listener : getListeners()) - { - try - { - errors.addAll(listener.canMove(c, newParent, user)); - } - catch (Exception e) - { - ExceptionUtil.logExceptionToMothership(null, new IllegalStateException(listener.getClass().getName() + ".canMove() threw an exception or violated @NotNull contract")); - } - } - if (!errors.isEmpty()) - { - ValidationException exception = new ValidationException(); - for (String error : errors) - { - exception.addError(new SimpleValidationError(error)); - } - throw exception; - } - - if (c.getParent().getId().equals(newParent.getId())) - return false; - - Container oldParent = c.getParent(); - Container oldProject = c.getProject(); - Container newProject = newParent.isRoot() ? c : newParent.getProject(); - - boolean changedProjects = !oldProject.getId().equals(newProject.getId()); - - // Synchronize the transaction, but not the listeners -- see #9901 - try (DbScope.Transaction t = ensureTransaction()) - { - new SqlExecutor(CORE.getSchema()).execute("UPDATE " + CORE.getTableInfoContainers() + " SET Parent = ? WHERE EntityId = ?", newParent.getId(), c.getId()); - - // Refresh the container directly from the database so the container reflects the new parent, isProject(), etc. - c = getForRowId(c.getRowId()); - - // this could be done in the trigger, but I prefer to put it in the transaction - if (changedProjects) - SecurityManager.changeProject(c, oldProject, newProject, user); - - clearCache(); - - try - { - ExperimentService.get().moveContainer(c, oldParent, newParent); - } - catch (ExperimentException e) - { - throw new RuntimeException(e); - } - - // Clear after the commit has propagated the state to other threads and transactions - // Do this in a commit task in case we've joined another existing DbScope.Transaction instead of starting our own - t.addCommitTask(() -> - { - clearCache(); - getChildrenMap(newParent); // reload the cache - }, DbScope.CommitTaskOption.POSTCOMMIT); - - t.commit(); - } - - Container newContainer = getForId(c.getId()); - fireMoveContainer(newContainer, oldParent, user); - - return changedProjects; - } - } - - public static void rename(@NotNull Container c, User user, String name) - { - rename(c, user, name, c.getTitle(), false); - } - - /** - * Transacted method to rename a container. Optionally, supports updating the title and aliasing the - * original container path when the name is changed (as name changes result in a new container path). - */ - public static Container rename(@NotNull Container c, User user, String name, @Nullable String title, boolean addAlias) - { - try (QuietCloser ignored = lockForMutation(MutatingOperation.rename, c); - DbScope.Transaction tx = ensureTransaction()) - { - final String oldName = c.getName(); - final String newName = StringUtils.trimToNull(name); - boolean isRenaming = !oldName.equals(newName); - StringBuilder errors = new StringBuilder(); - - // Rename - if (isRenaming) - { - // Issue 16221: Don't allow renaming of system reserved folders (e.g. /Shared, home, root, etc). - if (!isRenameable(c)) - throw new ApiUsageException("This folder may not be renamed as it is reserved by the system."); - - if (!Container.isLegalName(newName, c.isProject(), errors)) - throw new ApiUsageException(errors.toString()); - - // Issue 19061: Unable to do case-only container rename - if (c.getParent().hasChild(newName) && !c.equals(c.getParent().getChild(newName))) - { - if (c.getParent().isRoot()) - throw new ApiUsageException("The server already has a project with this name."); - throw new ApiUsageException("The " + (c.getParent().isProject() ? "project " : "folder ") + c.getParent().getPath() + " already has a folder with this name."); - } - - new SqlExecutor(CORE.getSchema()).execute("UPDATE " + CORE.getTableInfoContainers() + " SET Name=? WHERE EntityId=?", newName, c.getId()); - clearCache(); // Clear the entire cache, since containers cache their full paths - // Get new version since name has changed. - Container renamedContainer = getForId(c.getId()); - fireRenameContainer(renamedContainer, user, oldName); - // Clear again after the commit has propagated the state to other threads and transactions - // Do this in a commit task in case we've joined another existing DbScope.Transaction instead of starting our own - tx.addCommitTask(ContainerManager::clearCache, DbScope.CommitTaskOption.POSTCOMMIT); - - // Alias - if (addAlias) - { - // Intentionally use original container rather than the already renamedContainer - List newAliases = new ArrayList<>(getAliasesForContainer(c)); - newAliases.add(c.getPath()); - saveAliasesForContainer(c, newAliases, user); - } - } - - // Title - if (!c.getTitle().equals(title)) - { - if (!Container.isLegalTitle(title, errors)) - throw new ApiUsageException(errors.toString()); - updateTitle(c, title, user); - } - - tx.commit(); - } - catch (ValidationException e) - { - throw new IllegalArgumentException(e); - } - - return getForId(c.getId()); - } - - public static void setChildOrderToAlphabetical(Container parent) - { - setChildOrder(parent.getChildren(), true); - } - - public static void setChildOrder(Container parent, List orderedChildren) throws ContainerException - { - for (Container child : orderedChildren) - { - if (child == null || child.getParent() == null || !child.getParent().equals(parent)) // #13481 - throw new ContainerException("Invalid parent container of " + (child == null ? "null child container" : child.getPath())); - } - setChildOrder(orderedChildren, false); - } - - private static void setChildOrder(List siblings, boolean resetToAlphabetical) - { - try (DbScope.Transaction t = ensureTransaction()) - { - for (int index = 0; index < siblings.size(); index++) - { - Container current = siblings.get(index); - new SqlExecutor(CORE.getSchema()).execute("UPDATE " + CORE.getTableInfoContainers() + " SET SortOrder = ? WHERE EntityId = ?", - resetToAlphabetical ? 0 : index, current.getId()); - } - // Clear after the commit has propagated the state to other threads and transactions - // Do this in a commit task in case we've joined another existing DbScope.Transaction instead of starting our own - t.addCommitTask(ContainerManager::clearCache, DbScope.CommitTaskOption.POSTCOMMIT); - - t.commit(); - } - } - - private enum MutatingOperation - { - delete, - rename, - move - } - - private static final Map mutatingContainers = Collections.synchronizedMap(new IntHashMap<>()); - - private static QuietCloser lockForMutation(MutatingOperation op, Container c) - { - return lockForMutation(op, Collections.singletonList(c)); - } - - private static QuietCloser lockForMutation(MutatingOperation op, Collection containers) - { - List ids = new ArrayList<>(containers.size()); - synchronized (mutatingContainers) - { - for (Container container : containers) - { - MutatingOperation currentOp = mutatingContainers.get(container.getRowId()); - if (currentOp != null) - { - throw new ApiUsageException("Cannot start a " + op + " operation on " + container.getPath() + ". It is currently undergoing a " + currentOp); - } - ids.add(container.getRowId()); - } - ids.forEach(id -> mutatingContainers.put(id, op)); - } - return () -> - { - synchronized (mutatingContainers) - { - ids.forEach(mutatingContainers::remove); - } - }; - } - - // Delete containers from the database - private static boolean delete(final Collection containers, User user, @Nullable String comment) - { - // Do this check before we bother with any synchronization - for (Container container : containers) - { - if (!isDeletable(container)) - { - throw new ApiUsageException("Cannot delete container: " + container.getPath()); - } - } - - try (QuietCloser ignored = lockForMutation(MutatingOperation.delete, containers)) - { - boolean deleted = true; - for (Container c : containers) - { - deleted = deleted && delete(c, user, comment); - } - return deleted; - } - } - - // Delete a container from the database - private static boolean delete(final Container c, User user, @Nullable String comment) - { - // Verify method isn't called inappropriately - if (mutatingContainers.get(c.getRowId()) != MutatingOperation.delete) - { - throw new IllegalStateException("Container not flagged as being deleted: " + c.getPath()); - } - - LOG.debug("Starting container delete for " + c.getContainerNoun(true) + " " + c.getPath()); - - // Tell the search indexer to drop work for the container that's about to be deleted - SearchService.get().purgeForContainer(c); - - DbScope.RetryFn tryDeleteContainer = (tx) -> - { - // Verify that no children exist - Selector sel = new TableSelector(CORE.getTableInfoContainers(), new SimpleFilter(FieldKey.fromParts("Parent"), c), null); - - if (sel.exists()) - { - _removeFromCache(c, true); - return false; - } - - if (c.shouldRemoveFromPortal()) - { - // Need to remove portal page, too; container name is page's pageId and in container's parent container - Portal.PortalPage page = Portal.getPortalPage(c.getParent(), c.getName()); - if (null != page) // Be safe - Portal.deletePage(page); - - // Tell parent - setContainerTabDeleted(c.getParent(), c.getName(), c.getParent().getFolderType().getName()); - } - - fireDeleteContainer(c, user); - - SqlExecutor sqlExecutor = new SqlExecutor(CORE.getSchema()); - sqlExecutor.execute("DELETE FROM " + CORE.getTableInfoContainerAliases() + " WHERE ContainerRowId=?", c.getRowId()); - sqlExecutor.execute("DELETE FROM " + CORE.getTableInfoContainers() + " WHERE EntityId=?", c.getId()); - // now that the container is actually gone, delete all ACLs (better to have an ACL w/o object than object w/o ACL) - SecurityPolicyManager.removeAll(c); - // and delete all container-based sequences - DbSequenceManager.deleteAll(c); - - ExperimentService experimentService = ExperimentService.get(); - if (experimentService != null) - experimentService.removeContainerDataTypeExclusions(c.getId()); - - // After we've committed the transaction, be sure that we remove this container from the cache - // See https://www.labkey.org/issues/home/Developer/issues/details.view?issueId=17015 - tx.addCommitTask(() -> - { - // Be sure that we've waited until any threads that might be populating the cache have finished - // before we guarantee that we've removed this now-deleted container - DATABASE_QUERY_LOCK.lock(); - try - { - _removeFromCache(c, true); - } - finally - { - DATABASE_QUERY_LOCK.unlock(); - } - }, DbScope.CommitTaskOption.POSTCOMMIT); - String auditComment = c.getContainerNoun(true) + " " + c.getPath() + " was deleted"; - if (comment != null) - auditComment = auditComment.concat(". " + comment); - addAuditEvent(user, c, auditComment); - return true; - }; - - boolean success = CORE.getSchema().getScope().executeWithRetry(tryDeleteContainer); - if (success) - { - LOG.debug("Completed container delete for " + c.getContainerNoun(true) + " " + c.getPath()); - } - else - { - LOG.warn("Failed to delete container: " + c.getPath()); - } - return success; - } - - /** - * Delete a single container. Primarily for use by tests. - */ - public static boolean delete(final Container c, User user) - { - return delete(List.of(c), user, null); - } - - public static boolean isDeletable(Container c) - { - return !isSystemContainer(c); - } - - public static boolean isRenameable(Container c) - { - return !isSystemContainer(c); - } - - /** System containers include the root container, /Home, and /Shared */ - public static boolean isSystemContainer(Container c) - { - return c.equals(getRoot()) || c.equals(getHomeContainer()) || c.equals(getSharedContainer()); - } - - /** Has the container already been deleted or is it in the process of being deleted? */ - public static boolean exists(@Nullable Container c) - { - return c != null && null != getForId(c.getEntityId()) && mutatingContainers.get(c.getRowId()) != MutatingOperation.delete; - } - - public static void deleteAll(Container root, User user, @Nullable String comment) throws UnauthorizedException - { - if (!hasTreePermission(root, user, DeletePermission.class)) - throw new UnauthorizedException("You don't have delete permissions to all folders"); - - LOG.debug("Starting container (and children) delete for " + root.getContainerNoun(true) + " " + root.getPath()); - Set depthFirst = getAllChildrenDepthFirst(root); - depthFirst.add(root); - - delete(depthFirst, user, comment); - - LOG.debug("Completed container (and children) delete for " + root.getContainerNoun(true) + " " + root.getPath()); - } - - public static void deleteAll(Container root, User user) throws UnauthorizedException - { - deleteAll(root, user, null); - } - - private static void addAuditEvent(User user, Container c, String comment) - { - if (user != null) - { - AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, c, comment); - AuditLogService.get().addEvent(user, event); - } - } - - private static Set getAllChildrenDepthFirst(Container c) - { - Set set = new LinkedHashSet<>(); - getAllChildrenDepthFirst(c, set); - return set; - } - - private static void getAllChildrenDepthFirst(Container c, Collection list) - { - for (Container child : c.getChildren()) - { - getAllChildrenDepthFirst(child, list); - list.add(child); - } - } - - private static Container _getFromCachePath(Path path) - { - return CACHE_PATH.get(path); - } - - private static Container _addToCache(Container c) - { - assert DATABASE_QUERY_LOCK.isHeldByCurrentThread() : "Any cache modifications must be synchronized at a " + - "higher level so that we ensure that the container to be inserted still exists and hasn't been deleted"; - CACHE_ENTITY_ID.put(c.getEntityId(), c); - CACHE_PATH.put(c.getParsedPath(), c); - return c; - } - - private static void _clearChildrenFromCache(Container c) - { - CACHE_CHILDREN.remove(c.getEntityId()); - navTreeManageUncache(c); - } - - /** @param hierarchyChange whether the shape of the container tree has changed */ - private static void _removeFromCache(Container c, boolean hierarchyChange) - { - CACHE_ENTITY_ID.remove(c.getEntityId()); - CACHE_PATH.remove(c.getParsedPath()); - - if (hierarchyChange) - { - // This is strictly keeping track of the parent/child relationships themselves so it only needs to be - // cleared when the tree changes - CACHE_CHILDREN.clear(); - } - - navTreeManageUncache(c); - } - - public static void clearCache() - { - CACHE_PATH.clear(); - CACHE_ENTITY_ID.clear(); - CACHE_CHILDREN.clear(); - - // UNDONE: NavTreeManager should register a ContainerListener - NavTreeManager.uncacheAll(); - } - - private static void navTreeManageUncache(Container c) - { - // UNDONE: NavTreeManager should register a ContainerListener - NavTreeManager.uncacheTree(PROJECT_LIST_ID); - NavTreeManager.uncacheTree(getRoot().getId()); - - Container project = c.getProject(); - if (project != null) - { - NavTreeManager.uncacheTree(project.getId()); - NavTreeManager.uncacheTree(PROJECT_LIST_ID + project.getId()); - } - } - - public static void notifyContainerChange(String id, Property prop) - { - notifyContainerChange(id, prop, null); - } - - public static void notifyContainerChange(String id, Property prop, @Nullable User u) - { - if (_constructing.contains(new GUID(id))) - return; - - Container c = getForId(id); - if (null != c) - { - _removeFromCache(c, false); - c = getForId(id); // load a fresh container since the original might be stale. - if (null != c) - { - ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(c, u, prop, null, null); - firePropertyChangeEvent(evt); - } - } - } - - - /** Recursive, including root node */ - public static Set getAllChildren(Container root) - { - Set children = getAllChildrenDepthFirst(root); - children.add(root); - - return Collections.unmodifiableSet(children); - } - - /** - * Return all children of the root node, including root node, which have the given active module - */ - @NotNull - public static Set getAllChildrenWithModule(@NotNull Container root, @NotNull Module module) - { - Set children = new HashSet<>(); - for (Container candidate : getAllChildren(root)) - { - if (candidate.getActiveModules().contains(module)) - children.add(candidate); - } - return Collections.unmodifiableSet(children); - } - - public static long getContainerCount() - { - return new TableSelector(CORE.getTableInfoContainers()).getRowCount(); - } - - public static long getWorkbookCount() - { - return new TableSelector(CORE.getTableInfoContainers(), new SimpleFilter(FieldKey.fromParts("type"), "workbook"), null).getRowCount(); - } - - public static long getArchivedContainerCount() - { - return new TableSelector(CORE.getTableInfoContainers(), new SimpleFilter(FieldKey.fromParts("lockstate"), "Archived"), null).getRowCount(); - } - - public static long getAuditCommentRequiredCount() - { - SQLFragment sql = new SQLFragment( - "SELECT COUNT(*) FROM\n" + - " core.containers c\n" + - " JOIN prop.propertysets ps on c.entityid = ps.objectid\n" + - " JOIN prop.properties p on p.\"set\" = ps.\"set\"\n" + - "WHERE ps.category = '" + AUDIT_SETTINGS_PROPERTY_SET_NAME + "' AND p.name='"+ REQUIRE_USER_COMMENTS_PROPERTY_NAME + "' and p.value='true'"); - return new SqlSelector(CORE.getSchema(), sql).getObject(Long.class); - } - - - /** Retrieve entire container hierarchy */ - public static MultiValuedMap getContainerTree() - { - final MultiValuedMap mm = new ArrayListValuedHashMap<>(); - - // Get all containers and parents - SqlSelector selector = new SqlSelector(CORE.getSchema(), "SELECT Parent, EntityId FROM " + CORE.getTableInfoContainers() + " ORDER BY SortOrder, LOWER(Name) ASC"); - - selector.forEach(rs -> { - String parentId = rs.getString(1); - Container parent = (parentId != null ? getForId(parentId) : null); - Container child = getForId(rs.getString(2)); - - if (null != child) - mm.put(parent, child); - }); - - return mm; - } - - /** - * Returns a branch of the container tree including only the root and its descendants - * @param root The root container - * @return MultiMap of containers including root and its descendants - */ - public static MultiValuedMap getContainerTree(Container root) - { - //build a multimap of only the container ids - final MultiValuedMap mmIds = new ArrayListValuedHashMap<>(); - - // Get all containers and parents - Selector selector = new SqlSelector(CORE.getSchema(), "SELECT Parent, EntityId FROM " + CORE.getTableInfoContainers() + " ORDER BY SortOrder, LOWER(Name) ASC"); - - selector.forEach(rs -> mmIds.put(rs.getString(1), rs.getString(2))); - - //now find the root and build a MultiMap of it and its descendants - MultiValuedMap mm = new ArrayListValuedHashMap<>(); - mm.put(null, root); - addChildren(root, mmIds, mm); - return mm; - } - - private static void addChildren(Container c, MultiValuedMap mmIds, MultiValuedMap mm) - { - Collection childIds = mmIds.get(c.getId()); - if (null != childIds) - { - for (String childId : childIds) - { - Container child = getForId(childId); - if (null != child) - { - mm.put(c, child); - addChildren(child, mmIds, mm); - } - } - } - } - - public static Set getContainerSet(MultiValuedMap mm, User user, Class perm) - { - Collection containers = mm.values(); - if (null == containers) - return new HashSet<>(); - - return containers - .stream() - .filter(c -> c.hasPermission(user, perm)) - .collect(Collectors.toSet()); - } - - - public static SQLFragment getIdsAsCsvList(Set containers, SqlDialect d) - { - if (containers.isEmpty()) - return new SQLFragment("(NULL)"); // WHERE x IN (NULL) should match no rows - - SQLFragment csvList = new SQLFragment("("); - String comma = ""; - for (Container container : containers) - { - csvList.append(comma); - comma = ","; - csvList.appendValue(container, d); - } - csvList.append(")"); - - return csvList; - } - - - public static List getIds(User user, Class perm) - { - Set containers = getContainerSet(getContainerTree(), user, perm); - - List ids = new ArrayList<>(containers.size()); - - for (Container c : containers) - ids.add(c.getId()); - - return ids; - } - - - // - // ContainerListener - // - - public interface ContainerListener extends PropertyChangeListener - { - enum Order {First, Last} - - /** Called after a new container has been created */ - void containerCreated(Container c, User user); - - default void containerCreated(Container c, User user, @Nullable String auditMsg) - { - containerCreated(c, user); - } - - /** Called immediately prior to deleting the row from core.containers */ - void containerDeleted(Container c, User user); - - /** Called after the container has been moved to its new parent */ - void containerMoved(Container c, Container oldParent, User user); - - /** - * Called prior to moving a container, to find out if there are any issues that would prevent a successful move - * @return a list of errors that should prevent the move from happening, if any - */ - @NotNull - Collection canMove(Container c, Container newParent, User user); - - @Override - void propertyChange(PropertyChangeEvent evt); - } - - public static abstract class AbstractContainerListener implements ContainerListener - { - @Override - public void containerCreated(Container c, User user) - {} - - @Override - public void containerDeleted(Container c, User user) - {} - - @Override - public void containerMoved(Container c, Container oldParent, User user) - {} - - @NotNull - @Override - public Collection canMove(Container c, Container newParent, User user) - { - return Collections.emptyList(); - } - - @Override - public void propertyChange(PropertyChangeEvent evt) - {} - } - - - public static class ContainerPropertyChangeEvent extends PropertyChangeEvent implements PropertyChange - { - public final Property property; - public final Container container; - public User user; - - public ContainerPropertyChangeEvent(Container c, @Nullable User user, Property p, Object oldValue, Object newValue) - { - super(c, p.name(), oldValue, newValue); - container = c; - this.user = user; - property = p; - } - - public ContainerPropertyChangeEvent(Container c, Property p, Object oldValue, Object newValue) - { - this(c, null, p, oldValue, newValue); - } - - @Override - public Property getProperty() - { - return property; - } - } - - - // Thread-safe list implementation that allows iteration and modifications without external synchronization - private static final List _listeners = new CopyOnWriteArrayList<>(); - private static final List _laterListeners = new CopyOnWriteArrayList<>(); - - // These listeners are executed in the order they are registered, before the "Last" listeners - public static void addContainerListener(ContainerListener listener) - { - addContainerListener(listener, ContainerListener.Order.First); - } - - - // Explicitly request "Last" ordering via this method. "Last" listeners execute after all "First" listeners. - public static void addContainerListener(ContainerListener listener, ContainerListener.Order order) - { - if (ContainerListener.Order.First == order) - _listeners.add(listener); - else - _laterListeners.add(listener); - } - - - public static void removeContainerListener(ContainerListener listener) - { - _listeners.remove(listener); - _laterListeners.remove(listener); - } - - - private static List getListeners() - { - List combined = new ArrayList<>(_listeners.size() + _laterListeners.size()); - combined.addAll(_listeners); - combined.addAll(_laterListeners); - - return combined; - } - - - private static List getListenersReversed() - { - List combined = new LinkedList<>(); - - // Copy to guarantee consistency between .listIterator() and .size() - List copy = new ArrayList<>(_listeners); - ListIterator iter = copy.listIterator(copy.size()); - - // Iterate in reverse - while(iter.hasPrevious()) - combined.add(iter.previous()); - - // Copy to guarantee consistency between .listIterator() and .size() - // Add elements from the laterList in reverse order so that Core is fired last - List laterCopy = new ArrayList<>(_laterListeners); - ListIterator laterIter = laterCopy.listIterator(laterCopy.size()); - - // Iterate in reverse - while(laterIter.hasPrevious()) - combined.add(laterIter.previous()); - - return combined; - } - - - protected static void fireCreateContainer(Container c, User user, @Nullable String auditMsg) - { - List list = getListeners(); - - for (ContainerListener cl : list) - { - try - { - cl.containerCreated(c, user, auditMsg); - } - catch (Throwable t) - { - LOG.error("fireCreateContainer for " + cl.getClass().getName(), t); - } - } - } - - - protected static void fireDeleteContainer(Container c, User user) - { - List list = getListenersReversed(); - - for (ContainerListener l : list) - { - LOG.debug("Deleting " + c.getPath() + ": fireDeleteContainer for " + l.getClass().getName()); - try - { - l.containerDeleted(c, user); - } - catch (RuntimeException e) - { - LOG.error("fireDeleteContainer for " + l.getClass().getName(), e); - - // Fail fast (first Throwable aborts iteration), #17560 - throw e; - } - } - } - - - protected static void fireRenameContainer(Container c, User user, String oldValue) - { - ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(c, user, Property.Name, oldValue, c.getName()); - firePropertyChangeEvent(evt); - } - - - protected static void fireMoveContainer(Container c, Container oldParent, User user) - { - List list = getListeners(); - - for (ContainerListener cl : list) - { - // While we would ideally transact the full container move, that will likely cause long-blocking - // queries and/or deadlocks. For now, at least transact each separate move handler independently - try (DbScope.Transaction transaction = CoreSchema.getInstance().getSchema().getScope().ensureTransaction()) - { - cl.containerMoved(c, oldParent, user); - transaction.commit(); - } - } - ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(c, user, Property.Parent, oldParent, c.getParent()); - firePropertyChangeEvent(evt); - } - - - public static void firePropertyChangeEvent(ContainerPropertyChangeEvent evt) - { - if (_constructing.contains(evt.container.getEntityId())) - return; - - List list = getListeners(); - for (ContainerListener l : list) - { - try - { - l.propertyChange(evt); - } - catch (Throwable t) - { - LOG.error("firePropertyChangeEvent for " + l.getClass().getName(), t); - } - } - } - - private static final List MODULE_DEPENDENCY_PROVIDERS = new CopyOnWriteArrayList<>(); - - public static void registerModuleDependencyProvider(ModuleDependencyProvider provider) - { - MODULE_DEPENDENCY_PROVIDERS.add(provider); - } - - public static void forEachModuleDependencyProvider(Consumer action) - { - MODULE_DEPENDENCY_PROVIDERS.forEach(action); - } - - // Compliance module adds a locked project handler that checks permissions; without that, this implementation - // is used, and projects are never locked - static volatile LockedProjectHandler LOCKED_PROJECT_HANDLER = (project, user, contextualRoles, lockState) -> false; - - // Replaces any previously set LockedProjectHandler - public static void setLockedProjectHandler(LockedProjectHandler handler) - { - LOCKED_PROJECT_HANDLER = handler; - } - - public static Container createDefaultSupportContainer() - { - LOG.info("Creating default support container: " + DEFAULT_SUPPORT_PROJECT_PATH); - // create a "support" container. Admins can do anything, - // Users can read/write, Guests can read. - return bootstrapContainer(DEFAULT_SUPPORT_PROJECT_PATH, - RoleManager.getRole(AuthorRole.class), - RoleManager.getRole(ReaderRole.class) - ); - } - - public static void removeDefaultSupportContainer(User user) - { - Container support = getDefaultSupportContainer(); - if (support != null) - { - LOG.info("Removing default support container: " + DEFAULT_SUPPORT_PROJECT_PATH); - ContainerManager.delete(support, user); - } - } - - public static Container getDefaultSupportContainer() - { - return getForPath(DEFAULT_SUPPORT_PROJECT_PATH); - } - - public static List getAliasesForContainer(Container c) - { - return Collections.unmodifiableList(new SqlSelector(CORE.getSchema(), - new SQLFragment("SELECT Path FROM " + CORE.getTableInfoContainerAliases() + " WHERE ContainerRowId = ? ORDER BY Path", - c.getRowId())).getArrayList(String.class)); - } - - @Nullable - public static Container resolveContainerPathAlias(String path) - { - return resolveContainerPathAlias(path, false); - } - - @Nullable - private static Container resolveContainerPathAlias(String path, boolean top) - { - // Strip any trailing slashes - while (path.endsWith("/")) - { - path = path.substring(0, path.length() - 1); - } - - // Simple case -- resolve directly (sans alias) - Container aliased = getForPath(path); - if (aliased != null) - return aliased; - - // Simple case -- directly resolve from database - aliased = getForPathAlias(path); - if (aliased != null) - return aliased; - - // At the leaf and the container was not found - if (top) - return null; - - List splits = Arrays.asList(path.split("/")); - String subPath = ""; - for (int i=0; i < splits.size()-1; i++) // minus 1 due to leaving off last container - { - if (!splits.get(i).isEmpty()) - subPath += "/" + splits.get(i); - } - - aliased = resolveContainerPathAlias(subPath, false); - - if (aliased == null) - return null; - - String leafPath = aliased.getPath() + "/" + splits.get(splits.size()-1); - return resolveContainerPathAlias(leafPath, true); - } - - @Nullable - private static Container getForPathAlias(String path) - { - // We store the path as lower-case, so we don't need to also LOWER() on the value in core.ContainerAliases, letting the DB use the index - Container[] ret = new SqlSelector(CORE.getSchema(), - "SELECT * FROM " + CORE.getTableInfoContainers() + " c, " + CORE.getTableInfoContainerAliases() + " ca WHERE ca.ContainerRowId = c.RowId AND ca.path = LOWER(?)", - path).getArray(Container.class); - - return ret.length == 0 ? null : ret[0]; - } - - public static Container getMoveTargetContainer(@Nullable String queryName, @NotNull Container sourceContainer, User user, @Nullable String targetIdOrPath, Errors errors) - { - if (targetIdOrPath == null) - { - errors.reject(ERROR_GENERIC, "A target container must be specified for the move operation."); - return null; - } - - Container _targetContainer = getContainerForIdOrPath(targetIdOrPath); - if (_targetContainer == null) - { - errors.reject(ERROR_GENERIC, "The target container was not found: " + targetIdOrPath + "."); - return null; - } - - if (!_targetContainer.hasPermission(user, InsertPermission.class)) - { - String _queryName = queryName == null ? "this table" : "'" + queryName + "'"; - errors.reject(ERROR_GENERIC, "You do not have permission to move rows from " + _queryName + " to the target container: " + targetIdOrPath + "."); - return null; - } - - if (!isValidTargetContainer(sourceContainer, _targetContainer)) - { - errors.reject(ERROR_GENERIC, "Invalid target container for the move operation: " + targetIdOrPath + "."); - return null; - } - return _targetContainer; - } - - private static Container getContainerForIdOrPath(String targetContainer) - { - Container c = ContainerManager.getForId(targetContainer); - if (c == null) - c = ContainerManager.getForPath(targetContainer); - - return c; - } - - // targetContainer must be in the same app project at this time - // i.e. child of current project, project of current child, sibling within project - private static boolean isValidTargetContainer(Container current, Container target) - { - if (current.isRoot() || target.isRoot()) - return false; - - // Allow moving to the current container since we now allow the chosen entities to be from different containers - if (current.equals(target)) - return true; - - boolean moveFromProjectToChild = current.isProject() && target.getParent().equals(current); - boolean moveFromChildToProject = !current.isProject() && current.getParent().isProject() && current.getParent().equals(target); - boolean moveFromChildToSibling = !current.isProject() && current.getParent().isProject() && current.getParent().equals(target.getParent()); - - return moveFromProjectToChild || moveFromChildToProject || moveFromChildToSibling; - } - - public static int updateContainer(TableInfo dataTable, String idField, Collection ids, Container targetContainer, User user, boolean withModified) - { - try (DbScope.Transaction transaction = dataTable.getSchema().getScope().ensureTransaction()) - { - SQLFragment dataUpdate = new SQLFragment("UPDATE ").append(dataTable) - .append(" SET container = ").appendValue(targetContainer.getEntityId()); - if (withModified) - { - dataUpdate.append(", modified = ").appendValue(new Date()); - dataUpdate.append(", modifiedby = ").appendValue(user.getUserId()); - } - dataUpdate.append(" WHERE ").append(idField); - dataTable.getSchema().getSqlDialect().appendInClauseSql(dataUpdate, ids); - int numUpdated = new SqlExecutor(dataTable.getSchema()).execute(dataUpdate); - transaction.commit(); - - return numUpdated; - } - } - - /** - * If a container at the given path does not exist, create one and set permissions. If the container does exist, - * permissions are only set if there is no explicit ACL for the container. This prevents us from resetting - * permissions if all users are dropped. Implicitly done as an admin-level service user. - */ - @NotNull - public static Container bootstrapContainer(String path, @NotNull Role userRole, @Nullable Role guestRole) - { - Container c = null; - User user = User.getAdminServiceUser(); - - try - { - c = getForPath(path); - } - catch (RootContainerException e) - { - // Ignore this -- root doesn't exist yet - } - boolean newContainer = false; - - if (c == null) - { - LOG.debug("Creating new container for path '" + path + "'"); - newContainer = true; - c = ensureContainer(path, user); - } - - // Only set permissions if there are no explicit permissions - // set for this object or we just created it - Integer policyCount = null; - if (!newContainer) - { - policyCount = new SqlSelector(CORE.getSchema(), - "SELECT COUNT(*) FROM " + CORE.getTableInfoPolicies() + " WHERE ResourceId = ?", - c.getId()).getObject(Integer.class); - } - - if (newContainer || 0 == policyCount.intValue()) - { - LOG.debug("Setting permissions for '" + path + "'"); - MutableSecurityPolicy policy = new MutableSecurityPolicy(c); - policy.addRoleAssignment(SecurityManager.getGroup(Group.groupUsers), userRole); - if (guestRole != null) - policy.addRoleAssignment(SecurityManager.getGroup(Group.groupGuests), guestRole); - SecurityPolicyManager.savePolicy(policy, user); - } - - return c; - } - - /** - * @param container the container being created. May be null if we haven't actually created it yet - * @param parent the parent of the container being created. Used in case the container doesn't actually exist yet. - * @return the list of standard steps and any extra ones based on the container's FolderType - */ - public static List getCreateContainerWizardSteps(@Nullable Container container, @NotNull Container parent) - { - List navTrail = new ArrayList<>(); - - boolean isProject = parent.isRoot(); - - navTrail.add(new NavTree(isProject ? "Create Project" : "Create Folder")); - navTrail.add(new NavTree("Users / Permissions")); - if (isProject) - navTrail.add(new NavTree("Project Settings")); - if (container != null) - navTrail.addAll(container.getFolderType().getExtraSetupSteps(container)); - return navTrail; - } - - @TestTimeout(120) @TestWhen(TestWhen.When.BVT) - public static class TestCase extends Assert implements ContainerListener - { - Map _containers = new HashMap<>(); - Container _testRoot = null; - - @Before - public void setUp() - { - if (null == _testRoot) - { - Container junit = JunitUtil.getTestContainer(); - _testRoot = ensureContainer(junit, "ContainerManager$TestCase-" + GUID.makeGUID(), TestContext.get().getUser()); - addContainerListener(this); - } - } - - @After - public void tearDown() - { - removeContainerListener(this); - if (null != _testRoot) - deleteAll(_testRoot, TestContext.get().getUser()); - } - - @Test - public void testImproperFolderNamesBlocked() - { - String[] badNames = {"", "f\\o", "f/o", "f\\\\o", "foo;", "@foo", "foo" + '\u001F', '\u0000' + "foo", "fo" + '\u007F' + "o", "" + '\u009F'}; - - for (String name: badNames) - { - try - { - Container c = createContainer(_testRoot, name, TestContext.get().getUser()); - try - { - assertTrue(delete(c, TestContext.get().getUser())); - } - catch (Exception ignored) {} - fail("Should have thrown exception when trying to create container with name: " + name); - } - catch (ApiUsageException e) - { - // Do nothing, this is expected - } - } - } - - @Test - public void testCreateDeleteContainers() - { - int count = 20; - Random random = new Random(); - MultiValuedMap mm = new ArrayListValuedHashMap<>(); - - for (int i = 1; i <= count; i++) - { - int parentId = random.nextInt(i); - String parentName = 0 == parentId ? _testRoot.getName() : String.valueOf(parentId); - String childName = String.valueOf(i); - mm.put(parentName, childName); - } - - logNode(mm, _testRoot.getName(), 0); - for (int i=0; i<2; i++) //do this twice to make sure the containers were *really* deleted - { - createContainers(mm, _testRoot.getName(), _testRoot); - assertEquals(count, _containers.size()); - cleanUpChildren(mm, _testRoot.getName(), _testRoot); - assertEquals(0, _containers.size()); - } - } - - @Test - public void testCache() - { - assertEquals(0, _containers.size()); - assertEquals(0, getChildren(_testRoot).size()); - - Container one = createContainer(_testRoot, "one", TestContext.get().getUser()); - assertEquals(1, _containers.size()); - assertEquals(1, getChildren(_testRoot).size()); - assertEquals(0, getChildren(one).size()); - - Container oneA = createContainer(one, "A", TestContext.get().getUser()); - assertEquals(2, _containers.size()); - assertEquals(1, getChildren(_testRoot).size()); - assertEquals(1, getChildren(one).size()); - assertEquals(0, getChildren(oneA).size()); - - Container oneB = createContainer(one, "B", TestContext.get().getUser()); - assertEquals(3, _containers.size()); - assertEquals(1, getChildren(_testRoot).size()); - assertEquals(2, getChildren(one).size()); - assertEquals(0, getChildren(oneB).size()); - - Container deleteme = createContainer(one, "deleteme", TestContext.get().getUser()); - assertEquals(4, _containers.size()); - assertEquals(1, getChildren(_testRoot).size()); - assertEquals(3, getChildren(one).size()); - assertEquals(0, getChildren(deleteme).size()); - - assertTrue(delete(deleteme, TestContext.get().getUser())); - assertEquals(3, _containers.size()); - assertEquals(1, getChildren(_testRoot).size()); - assertEquals(2, getChildren(one).size()); - - Container oneC = createContainer(one, "C", TestContext.get().getUser()); - assertEquals(4, _containers.size()); - assertEquals(1, getChildren(_testRoot).size()); - assertEquals(3, getChildren(one).size()); - assertEquals(0, getChildren(oneC).size()); - - assertTrue(delete(oneC, TestContext.get().getUser())); - assertTrue(delete(oneB, TestContext.get().getUser())); - assertEquals(1, getChildren(one).size()); - - assertTrue(delete(oneA, TestContext.get().getUser())); - assertEquals(0, getChildren(one).size()); - - assertTrue(delete(one, TestContext.get().getUser())); - assertEquals(0, getChildren(_testRoot).size()); - assertEquals(0, _containers.size()); - } - - @Test - public void testFolderType() - { - // Test all folder types - List folderTypes = new ArrayList<>(FolderTypeManager.get().getAllFolderTypes()); - for (FolderType folderType : folderTypes) - { - if (!folderType.isProjectOnlyType()) // Dataspace can't be subfolder - testOneFolderType(folderType); - } - } - - private void testOneFolderType(FolderType folderType) - { - LOG.info("testOneFolderType(" + folderType.getName() + "): creating container"); - Container newFolder = createContainer(_testRoot, "folderTypeTest", TestContext.get().getUser()); - FolderType ft = newFolder.getFolderType(); - assertEquals(FolderType.NONE, ft); - - Container newFolderFromCache = getForId(newFolder.getId()); - assertNotNull(newFolderFromCache); - assertEquals(FolderType.NONE, newFolderFromCache.getFolderType()); - LOG.info("testOneFolderType(" + folderType.getName() + "): setting folder type"); - newFolder.setFolderType(folderType, TestContext.get().getUser()); - - newFolderFromCache = getForId(newFolder.getId()); - assertNotNull(newFolderFromCache); - assertEquals(newFolderFromCache.getFolderType().getName(), folderType.getName()); - assertEquals(newFolderFromCache.getFolderType().getDescription(), folderType.getDescription()); - - LOG.info("testOneFolderType(" + folderType.getName() + "): deleteAll"); - deleteAll(newFolder, TestContext.get().getUser()); // There might be subfolders because of container tabs - LOG.info("testOneFolderType(" + folderType.getName() + "): deleteAll complete"); - Container deletedContainer = getForId(newFolder.getId()); - - if (deletedContainer != null) - { - fail("Expected container with Id " + newFolder.getId() + " to be deleted, but found " + deletedContainer + ". Folder type was " + folderType); - } - } - - private static void createContainers(MultiValuedMap mm, String name, Container parent) - { - Collection nodes = mm.get(name); - - if (null == nodes) - return; - - for (String childName : nodes) - { - Container child = createContainer(parent, childName, TestContext.get().getUser()); - createContainers(mm, childName, child); - } - } - - private static void cleanUpChildren(MultiValuedMap mm, String name, Container parent) - { - Collection nodes = mm.get(name); - - if (null == nodes) - return; - - for (String childName : nodes) - { - Container child = getForPath(makePath(parent, childName)); - cleanUpChildren(mm, childName, child); - assertTrue(delete(child, TestContext.get().getUser())); - } - } - - private static void logNode(MultiValuedMap mm, String name, int offset) - { - Collection nodes = mm.get(name); - - if (null == nodes) - return; - - for (String childName : nodes) - { - LOG.debug(StringUtils.repeat(" ", offset) + childName); - logNode(mm, childName, offset + 1); - } - } - - // ContainerListener - @Override - public void propertyChange(PropertyChangeEvent evt) - { - } - - @Override - public void containerCreated(Container c, User user) - { - if (null == _testRoot || !c.getParsedPath().startsWith(_testRoot.getParsedPath())) - return; - _containers.put(c.getParsedPath(), c); - } - - - @Override - public void containerDeleted(Container c, User user) - { - _containers.remove(c.getParsedPath()); - } - - @Override - public void containerMoved(Container c, Container oldParent, User user) - { - } - - @NotNull - @Override - public Collection canMove(Container c, Container newParent, User user) - { - return Collections.emptyList(); - } - } - - static - { - ObjectFactory.Registry.register(Container.class, new ContainerFactory()); - } - - public static class ContainerFactory implements ObjectFactory - { - @Override - public Container fromMap(Map m) - { - throw new UnsupportedOperationException(); - } - - @Override - public Container fromMap(Container bean, Map m) - { - throw new UnsupportedOperationException(); - } - - @Override - public Map toMap(Container bean, Map m) - { - throw new UnsupportedOperationException(); - } - - @Override - public Container handle(ResultSet rs) throws SQLException - { - String id; - Container d; - String parentId = rs.getString("Parent"); - String name = rs.getString("Name"); - id = rs.getString("EntityId"); - int rowId = rs.getInt("RowId"); - int sortOrder = rs.getInt("SortOrder"); - Date created = rs.getTimestamp("Created"); - int createdBy = rs.getInt("CreatedBy"); - // _ts - String description = rs.getString("Description"); - String type = rs.getString("Type"); - String title = rs.getString("Title"); - boolean searchable = rs.getBoolean("Searchable"); - String lockStateString = rs.getString("LockState"); - LockState lockState = null != lockStateString ? Enums.getIfPresent(LockState.class, lockStateString).or(LockState.Unlocked) : LockState.Unlocked; - - LocalDate expirationDate = rs.getObject("ExpirationDate", LocalDate.class); - - // Could be running upgrade code before these recent columns have been added to the table. Use a find map - // to determine if they are present. Issue 51692. These checks could be removed after creation of these - // columns is incorporated into the bootstrap scripts. - Map findMap = ResultSetUtil.getFindMap(rs.getMetaData()); - Long fileRootSize = findMap.containsKey("FileRootSize") ? (Long)rs.getObject("FileRootSize") : null; // getObject() and cast because getLong() returns 0 for null - LocalDateTime fileRootLastCrawled = findMap.containsKey("FileRootLastCrawled") ? rs.getObject("FileRootLastCrawled", LocalDateTime.class) : null; - - Container dirParent = null; - if (null != parentId) - dirParent = getForId(parentId); - - d = new Container(dirParent, name, id, rowId, sortOrder, created, createdBy, searchable); - d.setDescription(description); - d.setType(type); - d.setTitle(title); - d.setLockState(lockState); - d.setExpirationDate(expirationDate); - d.setFileRootSize(fileRootSize); - d.setFileRootLastCrawled(fileRootLastCrawled); - return d; - } - - @Override - public ArrayList handleArrayList(ResultSet rs) throws SQLException - { - ArrayList list = new ArrayList<>(); - while (rs.next()) - { - list.add(handle(rs)); - } - return list; - } - } - - public static Container createFakeContainer(@Nullable String name, @Nullable Container parent) - { - return new Container(parent, name, GUID.makeGUID(), 1, 0, new Date(), 0, false); - } -} +/* + * Copyright (c) 2005-2018 Fred Hutchinson Cancer Research Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.data; + +import com.google.common.base.Enums; +import org.apache.commons.collections4.MultiValuedMap; +import org.apache.commons.collections4.multimap.ArrayListValuedHashMap; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.xmlbeans.XmlObject; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.labkey.api.Constants; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.admin.FolderExportContext; +import org.labkey.api.admin.FolderImportContext; +import org.labkey.api.admin.FolderImporterImpl; +import org.labkey.api.admin.FolderWriterImpl; +import org.labkey.api.admin.StaticLoggerGetter; +import org.labkey.api.attachments.AttachmentParent; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.provider.ContainerAuditProvider; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.ConcurrentHashSet; +import org.labkey.api.collections.IntHashMap; +import org.labkey.api.data.Container.ContainerException; +import org.labkey.api.data.Container.LockState; +import org.labkey.api.data.PropertyManager.WritablePropertyMap; +import org.labkey.api.data.SimpleFilter.InClause; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.data.validator.ColumnValidators; +import org.labkey.api.event.PropertyChange; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.module.FolderType; +import org.labkey.api.module.FolderTypeManager; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.portal.ProjectUrls; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.SimpleValidationError; +import org.labkey.api.query.ValidationException; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.Group; +import org.labkey.api.security.MutableSecurityPolicy; +import org.labkey.api.security.SecurityLogger; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.security.SecurityPolicyManager; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.CreateProjectPermission; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.InsertPermission; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.security.roles.AuthorRole; +import org.labkey.api.security.roles.ReaderRole; +import org.labkey.api.security.roles.Role; +import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.settings.AppProps; +import org.labkey.api.test.TestTimeout; +import org.labkey.api.test.TestWhen; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JunitUtil; +import org.labkey.api.util.MinorConfigurationException; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Path; +import org.labkey.api.util.QuietCloser; +import org.labkey.api.util.ReentrantLockWithName; +import org.labkey.api.util.ResultSetUtil; +import org.labkey.api.util.TestContext; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.FolderTab; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.NavTreeManager; +import org.labkey.api.view.Portal; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.view.ViewContext; +import org.labkey.api.writer.MemoryVirtualFile; +import org.labkey.folder.xml.FolderDocument; +import org.labkey.remoteapi.collections.CaseInsensitiveHashMap; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; + +import java.beans.PropertyChangeEvent; +import java.beans.PropertyChangeListener; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static org.labkey.api.action.SpringActionController.ERROR_GENERIC; + +/** + * This class manages a hierarchy of collections, backed by a database table called Containers. + * Containers are named using filesystem-like paths e.g. /proteomics/comet/. Each path + * maps to a UID and set of permissions. The current security scheme allows ACLs + * to be specified explicitly on the directory or completely inherited. ACLs are not combined. + *

+ * NOTE: we act like java.io.File(). Paths start with forward-slash, but do not end with forward-slash. + * The root container's name is '/'. This means that it is not always the case that + * me.getPath() == me.getParent().getPath() + "/" + me.getName() + *

+ * The synchronization goals are to keep invalid containers from creeping into the cache. For example, once + * a container is deleted, it should never get put back in the cache. We accomplish this by synchronizing on + * the removal from the cache, and the database lookup/cache insertion. While a container is in the middle + * of being deleted, it's OK for other clients to see it because FKs enforce that it's always internally + * consistent, even if some of the data has already been deleted. + */ +public class ContainerManager +{ + private static final Logger LOG = LogHelper.getLogger(ContainerManager.class, "Container (projects, folders, and workbooks) retrieval and management"); + private static final CoreSchema CORE = CoreSchema.getInstance(); + + private static final String PROJECT_LIST_ID = "Projects"; + + public static final String HOME_PROJECT_PATH = "/home"; + public static final String DEFAULT_SUPPORT_PROJECT_PATH = HOME_PROJECT_PATH + "/support"; + + private static final Cache CACHE_PATH = CacheManager.getCache(Constants.getMaxContainers(), CacheManager.DAY, "Containers by Path"); + private static final Cache CACHE_ENTITY_ID = CacheManager.getCache(Constants.getMaxContainers(), CacheManager.DAY, "Containers by EntityId"); + private static final Cache> CACHE_CHILDREN = CacheManager.getCache(Constants.getMaxContainers(), CacheManager.DAY, "Child EntityIds of Containers"); + private static final ReentrantLock DATABASE_QUERY_LOCK = new ReentrantLockWithName(ContainerManager.class, "DATABASE_QUERY_LOCK"); + public static final String FOLDER_TYPE_PROPERTY_SET_NAME = "folderType"; + public static final String FOLDER_TYPE_PROPERTY_NAME = "name"; + public static final String FOLDER_TYPE_PROPERTY_TABTYPE_OVERRIDDEN = "ctFolderTypeOverridden"; + public static final String TABFOLDER_CHILDREN_DELETED = "tabChildrenDeleted"; + public static final String AUDIT_SETTINGS_PROPERTY_SET_NAME = "containerAuditSettings"; + public static final String REQUIRE_USER_COMMENTS_PROPERTY_NAME = "requireUserComments"; + + private static final List _resourceProviders = new CopyOnWriteArrayList<>(); + + // containers that are being constructed, used to suppress events before fireCreateContainer() + private static final Set _constructing = new ConcurrentHashSet<>(); + + + /** enum of properties you can see in property change events */ + public enum Property + { + Name, + Parent, + Policy, + /** The default or active set of modules in the container has changed */ + Modules, + FolderType, + WebRoot, + AttachmentDirectory, + PipelineRoot, + Title, + Description, + SiteRoot, + StudyChange, + EndpointDirectory, + CloudStores + } + + static Path makePath(Container parent, String name) + { + if (null == parent) + return new Path(name); + return parent.getParsedPath().append(name, true); + } + + public static Container createMockContainer() + { + return new Container(null, "MockContainer", "01234567-8901-2345-6789-012345678901", 99999999, 0, new Date(), User.guest.getUserId(), true); + } + + private static Container createRoot() + { + Map m = new HashMap<>(); + m.put("Parent", null); + m.put("Name", ""); + Table.insert(null, CORE.getTableInfoContainers(), m); + + return getRoot(); + } + + private static DbScope.Transaction ensureTransaction() + { + return CORE.getSchema().getScope().ensureTransaction(DATABASE_QUERY_LOCK); + } + + private static int getNewChildSortOrder(Container parent) + { + int nextSortOrderVal = 0; + + List children = parent.getChildren(); + if (children != null) + { + for (Container child : children) + { + // find the max sort order value for the set of children + nextSortOrderVal = Math.max(nextSortOrderVal, child.getSortOrder()); + } + } + + // custom sorting applies: put new container at the end. + if (nextSortOrderVal > 0) + return nextSortOrderVal + 1; + + // we're sorted alphabetically + return 0; + } + + // TODO: Make private and force callers to use ensureContainer instead? + // TODO: Handle root creation here? + @NotNull + public static Container createContainer(Container parent, String name, @NotNull User user) + { + return createContainer(parent, name, null, null, NormalContainerType.NAME, user, null, null); + } + + public static final String WORKBOOK_DBSEQUENCE_NAME = "org.labkey.api.data.Workbooks"; + + // TODO: Pass in FolderType (separate from the container type of workbook, etc) and transact it with container creation? + @NotNull + public static Container createContainer(Container parent, String name, @Nullable String title, @Nullable String description, String type, @NotNull User user) + { + return createContainer(parent, name, title, description, type, user, null, null); + } + + @NotNull + public static Container createContainer(Container parent, String name, @Nullable String title, @Nullable String description, String type, @NotNull User user, @Nullable String auditMsg) + { + return createContainer(parent, name, title, description, type, user, auditMsg, null); + } + + @NotNull + public static Container createContainer(Container parent, String name, @Nullable String title, @Nullable String description, String type, @NotNull User user, @Nullable String auditMsg, + Consumer configureContainer) + { + ContainerType cType = ContainerTypeRegistry.get().getType(type); + if (cType == null) + throw new IllegalArgumentException("Unknown container type: " + type); + + // TODO: move this to ContainerType? + long sortOrder; + if (cType instanceof WorkbookContainerType) + { + sortOrder = DbSequenceManager.get(parent, WORKBOOK_DBSEQUENCE_NAME).next(); + + // Default workbook names are simply "" + if (name == null) + name = String.valueOf(sortOrder); + } + else + { + sortOrder = getNewChildSortOrder(parent); + } + + if (!parent.canHaveChildren()) + throw new IllegalArgumentException("Parent of a container must not be a " + parent.getContainerType().getName()); + + StringBuilder error = new StringBuilder(); + if (!Container.isLegalName(name, parent.isRoot(), error)) + throw new ApiUsageException(error.toString()); + + if (!Container.isLegalTitle(title, error)) + throw new ApiUsageException(error.toString()); + + Path path = makePath(parent, name); + SQLException sqlx = null; + Map insertMap = null; + + GUID entityId = new GUID(); + Container c; + + try + { + _constructing.add(entityId); + + try + { + Map m = new CaseInsensitiveHashMap<>(); + m.put("Parent", parent.getId()); + m.put("Name", name); + m.put("Title", title); + m.put("SortOrder", sortOrder); + m.put("EntityId", entityId); + if (null != description) + m.put("Description", description); + m.put("Type", type); + insertMap = Table.insert(user, CORE.getTableInfoContainers(), m); + } + catch (RuntimeSQLException x) + { + if (!x.isConstraintException()) + throw x; + sqlx = x.getSQLException(); + } + + _clearChildrenFromCache(parent); + + c = insertMap == null ? null : getForId(entityId); + + if (null == c) + { + if (null != sqlx) + throw new RuntimeSQLException(sqlx); + else + throw new RuntimeException("Container for path '" + path + "' was not created properly."); + } + + User savePolicyUser = user; + if (c.isProject() && !c.hasPermission(user, AdminPermission.class) && ContainerManager.getRoot().hasPermission(user, CreateProjectPermission.class)) + { + // Special case for project creators who don't necessarily yet have permission to save the policy of + // the project they just created + savePolicyUser = User.getAdminServiceUser(); + } + + // Workbooks inherit perms from their parent so don't create a policy if this is a workbook + if (c.isContainerFor(ContainerType.DataType.permissions)) + { + SecurityManager.setAdminOnlyPermissions(c, savePolicyUser); + } + + _removeFromCache(c, true); // seems odd, but it removes c.getProject() which clears other things from the cache + + // Initialize the list of active modules in the Container + c.getActiveModules(true, true, user); + + if (c.isProject()) + { + SecurityManager.createNewProjectGroups(c, savePolicyUser); + } + else + { + // If current user does NOT have admin permission on this container or the project has been + // explicitly set to have new subfolders inherit permissions, then inherit permissions + // (otherwise they would not be able to see the folder) + boolean hasAdminPermission = c.hasPermission(user, AdminPermission.class); + if ((!hasAdminPermission && !user.hasRootAdminPermission()) || SecurityManager.shouldNewSubfoldersInheritPermissions(c.getProject())) + SecurityManager.setInheritPermissions(c); + } + + // NOTE parent caches some info about children (e.g. hasWorkbookChildren) + // since mutating cached objects is frowned upon, just uncache parent + // CONSIDER: we could perhaps only uncache if the child is a workbook, but I think this reasonable + _removeFromCache(parent, true); + + if (null != configureContainer) + configureContainer.accept(c); + } + finally + { + _constructing.remove(entityId); + } + + fireCreateContainer(c, user, auditMsg); + + return c; + } + + public static void addSecurableResourceProvider(ContainerSecurableResourceProvider provider) + { + _resourceProviders.add(provider); + } + + public static List getSecurableResourceProviders() + { + return Collections.unmodifiableList(_resourceProviders); + } + + public static Container createContainerFromTemplate(Container parent, String name, String title, Container templateContainer, User user, FolderExportContext exportCtx, Consumer afterCreateHandler) throws Exception + { + MemoryVirtualFile vf = new MemoryVirtualFile(); + + // export objects from the source template folder + FolderWriterImpl writer = new FolderWriterImpl(); + writer.write(templateContainer, exportCtx, vf); + + // create the new target container + Container c = createContainer(parent, name, title, null, NormalContainerType.NAME, user, null, afterCreateHandler); + + // import objects into the target folder + XmlObject folderXml = vf.getXmlBean("folder.xml"); + if (folderXml instanceof FolderDocument folderDoc) + { + FolderImportContext importCtx = new FolderImportContext(user, c, folderDoc, null, new StaticLoggerGetter(LogManager.getLogger(FolderImporterImpl.class)), vf); + + FolderImporterImpl importer = new FolderImporterImpl(); + importer.process(null, importCtx, vf); + } + + return c; + } + + public static void setRequireAuditComments(Container container, User user, @NotNull Boolean required) + { + WritablePropertyMap props = PropertyManager.getWritableProperties(container, AUDIT_SETTINGS_PROPERTY_SET_NAME, true); + String originalValue = props.get(REQUIRE_USER_COMMENTS_PROPERTY_NAME); + props.put(REQUIRE_USER_COMMENTS_PROPERTY_NAME, required.toString()); + props.save(); + + addAuditEvent(user, container, + "Changed " + REQUIRE_USER_COMMENTS_PROPERTY_NAME + " from \"" + + originalValue + "\" to \"" + required + "\""); + } + + public static void setFolderType(Container c, FolderType folderType, User user, BindException errors) + { + FolderType oldType = c.getFolderType(); + + if (folderType.equals(oldType)) + return; + + List errorStrings = new ArrayList<>(); + + if (!c.isProject() && folderType.isProjectOnlyType()) + errorStrings.add("Cannot set a subfolder to " + folderType.getName() + " because it is a project-only folder type."); + + // Check for any containers that need to be moved into container tabs + if (errorStrings.isEmpty() && folderType.hasContainerTabs()) + { + List childTabFoldersNonMatchingTypes = new ArrayList<>(); + List containersBecomingTabs = findAndCheckContainersMatchingTabs(c, folderType, childTabFoldersNonMatchingTypes, errorStrings); + + if (errorStrings.isEmpty()) + { + if (!containersBecomingTabs.isEmpty()) + { + // Make containers tab container; Folder tab will find them by name + try (DbScope.Transaction transaction = ensureTransaction()) + { + for (Container container : containersBecomingTabs) + updateType(container, TabContainerType.NAME, user); + + transaction.commit(); + } + } + + // Check these and change type unless they were overridden explicitly + for (Container container : childTabFoldersNonMatchingTypes) + { + if (!isContainerTabTypeOverridden(container)) + { + FolderTab newTab = folderType.findTab(container.getName()); + assert null != newTab; // There must be a tab because it caused the container to get into childTabFoldersNonMatchingTypes + FolderType newType = newTab.getFolderType(); + if (null == newType) + newType = FolderType.NONE; // default to NONE + setFolderType(container, newType, user, errors); + } + } + } + } + + if (errorStrings.isEmpty()) + { + oldType.unconfigureContainer(c, user); + WritablePropertyMap props = PropertyManager.getWritableProperties(c, FOLDER_TYPE_PROPERTY_SET_NAME, true); + props.put(FOLDER_TYPE_PROPERTY_NAME, folderType.getName()); + + if (c.isContainerTab()) + { + boolean containerTabTypeOverridden = false; + FolderTab tab = c.getParent().getFolderType().findTab(c.getName()); + if (null != tab && !folderType.equals(tab.getFolderType())) + containerTabTypeOverridden = true; + props.put(FOLDER_TYPE_PROPERTY_TABTYPE_OVERRIDDEN, Boolean.toString(containerTabTypeOverridden)); + } + props.save(); + + notifyContainerChange(c.getId(), Property.FolderType, user); + folderType.configureContainer(c, user); // Configure new only after folder type has been changed + + // TODO: Not needed? I don't think we've changed the container's state. + _removeFromCache(c, false); + } + else + { + for (String errorString : errorStrings) + errors.reject(SpringActionController.ERROR_MSG, errorString); + } + } + + public static void checkContainerValidity(Container c) throws ContainerException + { + // Check container for validity; in rare cases user may have changed their custom folderType.xml and caused + // duplicate subfolders (same name) to exist + // Get list of child containers that are not container tabs, but match container tabs; these are bad + FolderType folderType = getFolderType(c); + List errorStrings = new ArrayList<>(); + List childTabFoldersNonMatchingTypes = new ArrayList<>(); + List containersMatchingTabs = findAndCheckContainersMatchingTabs(c, folderType, childTabFoldersNonMatchingTypes, errorStrings); + if (!containersMatchingTabs.isEmpty()) + { + throw new Container.ContainerException("Folder " + c.getPath() + + " has a subfolder with the same name as a container tab folder, which is an invalid state." + + " This may have been caused by changing the folder type's tabs after this folder was set to its folder type." + + " An administrator should either delete the offending subfolder or change the folder's folder type.\n"); + } + } + + public static List findAndCheckContainersMatchingTabs(Container c, FolderType folderType, + List childTabFoldersNonMatchingTypes, List errorStrings) + { + List containersMatchingTabs = new ArrayList<>(); + for (FolderTab folderTab : folderType.getDefaultTabs()) + { + if (folderTab.getTabType() == FolderTab.TAB_TYPE.Container) + { + for (Container child : c.getChildren()) + { + if (child.getName().equalsIgnoreCase(folderTab.getName())) + { + if (!child.getFolderType().getName().equalsIgnoreCase(folderTab.getFolderTypeName())) + { + if (child.isContainerTab()) + childTabFoldersNonMatchingTypes.add(child); // Tab type doesn't match child tab folder + else + errorStrings.add("Child folder " + child.getName() + + " matches container tab, but folder type " + child.getFolderType().getName() + " doesn't match tab's folder type " + + folderTab.getFolderTypeName() + "."); + } + + int childCount = child.getChildren().size(); + if (childCount > 0) + { + errorStrings.add("Child folder " + child.getName() + + " matches container tab, but cannot be converted to a tab folder because it has " + childCount + " children."); + } + + if (!child.isConvertibleToTab()) + { + errorStrings.add("Child folder " + child.getName() + + " matches container tab, but cannot be converted to a tab folder because it is a " + child.getContainerNoun() + "."); + } + + if (!child.isContainerTab()) + containersMatchingTabs.add(child); + + break; // we found name match; can't be another + } + } + } + } + return containersMatchingTabs; + } + + private static final Set containersWithBadFolderTypes = new ConcurrentHashSet<>(); + + @NotNull + public static FolderType getFolderType(Container c) + { + String name = getFolderTypeName(c); + FolderType folderType; + + if (null != name) + { + folderType = FolderTypeManager.get().getFolderType(name); + + if (null == folderType) + { + // If we're upgrading then folder types won't be defined yet... don't warn in that case. + if (!ModuleLoader.getInstance().isUpgradeInProgress() && + !ModuleLoader.getInstance().isUpgradeRequired() && + !containersWithBadFolderTypes.contains(c)) + { + LOG.warn("No such folder type " + name + " for folder " + c.toString()); + containersWithBadFolderTypes.add(c); + } + + folderType = FolderType.NONE; + } + } + else + folderType = FolderType.NONE; + + return folderType; + } + + /** + * Most code should call getFolderType() instead. + * Useful for finding the name of the folder type BEFORE startup is complete, so the FolderType itself + * may not be available. + */ + @Nullable + public static String getFolderTypeName(Container c) + { + Map props = PropertyManager.getProperties(c, FOLDER_TYPE_PROPERTY_SET_NAME); + return props.get(FOLDER_TYPE_PROPERTY_NAME); + } + + + @NotNull + public static Map getFolderTypeNameContainerCounts(Container root) + { + Map nameCounts = new TreeMap<>(); + for (Container c : getAllChildren(root)) + { + Integer count = nameCounts.get(c.getFolderType().getName()); + if (null == count) + { + count = Integer.valueOf(0); + } + nameCounts.put(c.getFolderType().getName(), ++count); + } + return nameCounts; + } + + @NotNull + public static Map getProductFoldersMetrics(@NotNull FolderType folderType) + { + Container root = getRoot(); + Map metrics = new TreeMap<>(); + List counts = new ArrayList<>(); + for (Container c : root.getChildren()) + { + if (!c.getFolderType().getName().equals(folderType.getName())) + continue; + + int childCount = c.getChildren().stream().filter(Container::isInFolderNav).toList().size(); + counts.add(childCount); + } + + int totalFolderTypeMatch = counts.size(); + if (totalFolderTypeMatch == 0) + return metrics; + + Collections.sort(counts); + int median = counts.get((totalFolderTypeMatch - 1)/2); + if (totalFolderTypeMatch % 2 == 0 ) + { + int low = counts.get(totalFolderTypeMatch/2 - 1); + int high = counts.get(totalFolderTypeMatch/2); + median = Math.round((low + high) / 2.0f); + } + int maxProjectsCount = counts.get(totalFolderTypeMatch - 1); + int totalProjectsCount = counts.stream().mapToInt(Integer::intValue).sum(); + int averageProjectsCount = Math.round((float) totalProjectsCount /totalFolderTypeMatch); + + metrics.put("totalSubProjectsCount", totalProjectsCount); + metrics.put("averageSubProjectsPerHomeProject", averageProjectsCount); + metrics.put("medianSubProjectsCountPerHomeProject", median); + metrics.put("maxSubProjectsCountInHomeProject", maxProjectsCount); + + return metrics; + } + + public static boolean isContainerTabTypeThisOrChildrenOverridden(Container c) + { + if (isContainerTabTypeOverridden(c)) + return true; + if (c.getFolderType().hasContainerTabs()) + { + for (Container child : c.getChildren()) + { + if (child.isContainerTab() && isContainerTabTypeOverridden(child)) + return true; + } + } + return false; + } + + public static boolean isContainerTabTypeOverridden(Container c) + { + Map props = PropertyManager.getProperties(c, FOLDER_TYPE_PROPERTY_SET_NAME); + String overridden = props.get(FOLDER_TYPE_PROPERTY_TABTYPE_OVERRIDDEN); + return (null != overridden) && overridden.equalsIgnoreCase("true"); + } + + private static void setContainerTabDeleted(Container c, String tabName, String folderTypeName) + { + // Add prop in this category + WritablePropertyMap props = PropertyManager.getWritableProperties(c, TABFOLDER_CHILDREN_DELETED, true); + props.put(getDeletedTabKey(tabName, folderTypeName), "true"); + props.save(); + } + + public static void clearContainerTabDeleted(Container c, String tabName, String folderTypeName) + { + WritablePropertyMap props = PropertyManager.getWritableProperties(c, TABFOLDER_CHILDREN_DELETED, true); + String key = getDeletedTabKey(tabName, folderTypeName); + if (props.containsKey(key)) + { + props.remove(key); + props.save(); + } + } + + public static boolean hasContainerTabBeenDeleted(Container c, String tabName, String folderTypeName) + { + // We keep arbitrary number of deleted children tabs using suffix 0, 1, 2.... + Map props = PropertyManager.getProperties(c, TABFOLDER_CHILDREN_DELETED); + return props.containsKey(getDeletedTabKey(tabName, folderTypeName)); + } + + private static String getDeletedTabKey(String tabName, String folderTypeName) + { + return tabName + "-TABDELETED-FOLDER-" + folderTypeName; + } + + @NotNull + public static Container ensureContainer(@NotNull String path, @NotNull User user) + { + return ensureContainer(Path.parse(path), user); + } + + @NotNull + public static Container ensureContainer(@NotNull Path path, @NotNull User user) + { + Container c = null; + + try + { + c = getForPath(path); + } + catch (RootContainerException e) + { + // Ignore this -- root doesn't exist yet + } + + if (null == c) + { + if (path.isEmpty()) + c = createRoot(); + else + { + Path parentPath = path.getParent(); + c = ensureContainer(parentPath, user); + c = createContainer(c, path.getName(), null, null, NormalContainerType.NAME, user); + } + } + return c; + } + + + @NotNull + public static Container ensureContainer(Container parent, String name, User user) + { + // NOTE: Running outside a tx doesn't seem to be necessary. +// if (CORE.getSchema().getScope().isTransactionActive()) +// throw new IllegalStateException("Transaction should not be active"); + + Container c = null; + + try + { + c = getForPath(makePath(parent,name)); + } + catch (RootContainerException e) + { + // Ignore this -- root doesn't exist yet + } + + if (null == c) + { + c = createContainer(parent, name, user); + } + return c; + } + + public static void updateDescription(Container container, String description, User user) + throws ValidationException + { + ColumnValidators.validate(CORE.getTableInfoContainers().getColumn("Title"), null, 1, description); + + //For some reason, there is no primary key defined on core.containers + //so we can't use Table.update here + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + sql.append(" SET Description=? WHERE RowID=?").add(description).add(container.getRowId()); + new SqlExecutor(CORE.getSchema()).execute(sql); + + String oldValue = container.getDescription(); + _removeFromCache(container, false); + container = getForRowId(container.getRowId()); + ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(container, user, Property.Description, oldValue, description); + firePropertyChangeEvent(evt); + } + + public static void updateSearchable(Container container, boolean searchable, User user) + { + //For some reason, there is no primary key defined on core.containers + //so we can't use Table.update here + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + sql.append(" SET Searchable=? WHERE RowID=?").add(searchable).add(container.getRowId()); + new SqlExecutor(CORE.getSchema()).execute(sql); + + _removeFromCache(container, false); + } + + public static void updateLockState(Container container, LockState lockState, @NotNull Runnable auditRunnable) + { + //For some reason there is no primary key defined on core.containers + //so we can't use Table.update here + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + sql.append(" SET LockState = ?, ExpirationDate = NULL WHERE RowID = ?").add(lockState).add(container.getRowId()); + new SqlExecutor(CORE.getSchema()).execute(sql); + + _removeFromCache(container, false); + + auditRunnable.run(); + } + + public static List getExcludedProjects() + { + return getProjects().stream() + .filter(p->p.getLockState() == Container.LockState.Excluded) + .collect(Collectors.toList()); + } + + public static List getNonExcludedProjects() + { + return getProjects().stream() + .filter(p->p.getLockState() != Container.LockState.Excluded) + .collect(Collectors.toList()); + } + + public static void setExcludedProjects(Collection ids, @NotNull Runnable auditRunnable) + { + // First clear all existing "Excluded" states + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + sql.append(" SET LockState = NULL, ExpirationDate = NULL WHERE LockState = ?").add(LockState.Excluded); + new SqlExecutor(CORE.getSchema()).execute(sql); + + // Now set the passed-in projects to "Excluded" + if (!ids.isEmpty()) + { + ColumnInfo entityIdCol = CORE.getTableInfoContainers().getColumn("EntityId"); + Filter inClauseFilter = new SimpleFilter(new InClause(entityIdCol.getFieldKey(), ids)); + SQLFragment frag = new SQLFragment("UPDATE "); + frag.append(CORE.getTableInfoContainers().getSelectName()); + frag.append(" SET LockState = ?, ExpirationDate = NULL "); + frag.add(LockState.Excluded); + frag.append(inClauseFilter.getSQLFragment(CORE.getSqlDialect(), "c", Map.of(entityIdCol.getFieldKey(), entityIdCol))); + new SqlExecutor(CORE.getSchema()).execute(frag); + } + + clearCache(); + + auditRunnable.run(); + } + + public static void archiveContainer(User user, Container container, boolean archive) + { + if (container.isRoot() || container.isProject() || container.isAppHomeFolder()) + throw new ApiUsageException("Archive action not supported for this folder."); + + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers().getSelectName()); + if (archive) + { + sql.append(" SET LockState = ? "); + sql.add(LockState.Archived); + sql.append(" WHERE LockState IS NULL "); + } + else + { + sql.append(" SET LockState = NULL WHERE LockState = ? "); + sql.add(LockState.Archived); + } + sql.append("AND EntityId = ? "); + sql.add(container.getEntityId()); + new SqlExecutor(CORE.getSchema()).execute(sql); + + clearCache(); + + addAuditEvent(user, container, archive ? "Container has been archived." : "Archived container has been restored."); + } + + public static void updateExpirationDate(Container container, LocalDate expirationDate, @NotNull Runnable auditRunnable) + { + //For some reason there is no primary key defined on core.containers + //so we can't use Table.update here + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + // Note: jTDS doesn't support LocalDate, so convert to java.sql.Date + sql.append(" SET ExpirationDate = ? WHERE RowID = ?").add(java.sql.Date.valueOf(expirationDate)).add(container.getRowId()); + + new SqlExecutor(CORE.getSchema()).execute(sql); + + _removeFromCache(container, false); + + auditRunnable.run(); + } + + public static void updateType(Container container, String newType, User user) + { + //For some reason there is no primary key defined on core.containers + //so we can't use Table.update here + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + sql.append(" SET Type=? WHERE RowID=?").add(newType).add(container.getRowId()); + new SqlExecutor(CORE.getSchema()).execute(sql); + + _removeFromCache(container, false); + } + + public static void updateTitle(Container container, String title, User user) + throws ValidationException + { + ColumnValidators.validate(CORE.getTableInfoContainers().getColumn("Title"), null, 1, title); + + //For some reason there is no primary key defined on core.containers + //so we can't use Table.update here + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + sql.append(" SET Title=? WHERE RowID=?").add(title).add(container.getRowId()); + new SqlExecutor(CORE.getSchema()).execute(sql); + + _removeFromCache(container, false); + String oldValue = container.getTitle(); + container = getForRowId(container.getRowId()); + ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(container, user, Property.Title, oldValue, title); + firePropertyChangeEvent(evt); + } + + public static void uncache(Container c) + { + _removeFromCache(c, true); + } + + public static final String SHARED_CONTAINER_PATH = "/Shared"; + + @NotNull + public static Container getSharedContainer() + { + return ensureContainer(Path.parse(SHARED_CONTAINER_PATH), User.getAdminServiceUser()); + } + + public static List getChildren(Container parent) + { + return new ArrayList<>(getChildrenMap(parent).values()); + } + + // Default is to include all types of children, as seems only appropriate + public static List getChildren(Container parent, User u, Class perm) + { + return getChildren(parent, u, perm, null, ContainerTypeRegistry.get().getTypeNames()); + } + + public static List getChildren(Container parent, User u, Class perm, Set roles) + { + return getChildren(parent, u, perm, roles, ContainerTypeRegistry.get().getTypeNames()); + } + + public static List getChildren(Container parent, User u, Class perm, String typeIncluded) + { + return getChildren(parent, u, perm, null, Collections.singleton(typeIncluded)); + } + + public static List getChildren(Container parent, User u, Class perm, Set roles, Set includedTypes) + { + List children = new ArrayList<>(); + for (Container child : getChildrenMap(parent).values()) + if (includedTypes.contains(child.getContainerType().getName()) && child.hasPermission(u, perm, roles)) + children.add(child); + + return children; + } + + public static List getAllChildren(Container parent, User u) + { + return getAllChildren(parent, u, ReadPermission.class, null, ContainerTypeRegistry.get().getTypeNames()); + } + + public static List getAllChildren(Container parent, User u, Class perm) + { + return getAllChildren(parent, u, perm, null, ContainerTypeRegistry.get().getTypeNames()); + } + + // Default is to include all types of children + public static List getAllChildren(Container parent, User u, Class perm, Set roles) + { + return getAllChildren(parent, u, perm, roles, ContainerTypeRegistry.get().getTypeNames()); + } + + public static List getAllChildren(Container parent, User u, Class perm, String typeIncluded) + { + return getAllChildren(parent, u, perm, null, Collections.singleton(typeIncluded)); + } + + public static List getAllChildren(Container parent, User u, Class perm, Set roles, Set typesIncluded) + { + Set allChildren = getAllChildren(parent); + List result = new ArrayList<>(allChildren.size()); + + for (Container container : allChildren) + { + if (typesIncluded.contains(container.getContainerType().getName()) && container.hasPermission(u, perm, roles)) + { + result.add(container); + } + } + + return result; + } + + // Returns the next available child container name based on the baseName + public static String getAvailableChildContainerName(Container c, String baseName) + { + List children = getChildren(c); + Map folders = new HashMap<>(children.size() * 2); + for (Container child : children) + folders.put(child.getName(), child); + + String availableContainerName = baseName; + int i = 1; + while (folders.containsKey(availableContainerName)) + { + availableContainerName = baseName + " " + i++; + } + + return availableContainerName; + } + + // Returns true only if user has the specified permission in the entire container tree starting at root + public static boolean hasTreePermission(Container root, User u, Class perm) + { + for (Container c : getAllChildren(root)) + if (!c.hasPermission(u, perm)) + return false; + + return true; + } + + private static Map getChildrenMap(Container parent) + { + if (!parent.canHaveChildren()) + { + // Optimization to avoid database query (important because some installs have tens of thousands of + // workbooks) when the container is a workbook, which is not allowed to have children + return Collections.emptyMap(); + } + + List childIds = CACHE_CHILDREN.get(parent.getEntityId()); + if (null == childIds) + { + try (DbScope.Transaction t = ensureTransaction()) + { + List children = new SqlSelector(CORE.getSchema(), + "SELECT * FROM " + CORE.getTableInfoContainers() + " WHERE Parent = ? ORDER BY SortOrder, LOWER(Name)", + parent.getId()).getArrayList(Container.class); + + childIds = new ArrayList<>(children.size()); + for (Container c : children) + { + childIds.add(c.getEntityId()); + _addToCache(c); + } + childIds = Collections.unmodifiableList(childIds); + CACHE_CHILDREN.put(parent.getEntityId(), childIds); + // No database changes to commit, but need to decrement the transaction counter + t.commit(); + } + } + + if (childIds.isEmpty()) + return Collections.emptyMap(); + + // Use a LinkedHashMap to preserve the order defined by the user - they're not necessarily alphabetical + Map ret = new LinkedHashMap<>(); + for (GUID id : childIds) + { + Container c = getForId(id); + if (null != c) + ret.put(c.getName(), c); + } + return Collections.unmodifiableMap(ret); + } + + public static Container getForRowId(int id) + { + Selector selector = new SqlSelector(CORE.getSchema(), new SQLFragment("SELECT * FROM " + CORE.getTableInfoContainers() + " WHERE RowId = ?", id)); + return selector.getObject(Container.class); + } + + public static @Nullable Container getForId(@NotNull GUID guid) + { + return guid != null ? getForId(guid.toString()) : null; + } + + public static @Nullable Container getForId(@Nullable String id) + { + //if the input string is not a GUID, just return null, + //so that we don't get a SQLException when the database + //tries to convert it to a unique identifier. + if (!GUID.isGUID(id)) + return null; + + GUID guid = new GUID(id); + + Container d = CACHE_ENTITY_ID.get(guid); + if (null != d) + return d; + + try (DbScope.Transaction t = ensureTransaction()) + { + Container result = new SqlSelector( + CORE.getSchema(), + "SELECT * FROM " + CORE.getTableInfoContainers() + " WHERE EntityId = ?", + id).getObject(Container.class); + if (result != null) + { + result = _addToCache(result); + } + // No database changes to commit, but need to decrement the counter + t.commit(); + + return result; + } + } + + public static Container getChild(Container c, String name) + { + Path path = c.getParsedPath().append(name); + + Container d = _getFromCachePath(path); + if (null != d) + return d; + + Map map = getChildrenMap(c); + return map.get(name); + } + + + public static Container getForURL(@NotNull ActionURL url) + { + Container ret = getForPath(url.getExtraPath()); + if (ret == null) + ret = getForId(StringUtils.strip(url.getExtraPath(), "/")); + return ret; + } + + + public static Container getForPath(@NotNull String path) + { + if (GUID.isGUID(path)) + { + Container c = getForId(path); + if (c != null) + return c; + } + + Path p = Path.parse(path); + return getForPath(p); + } + + public static Container getForPath(Path path) + { + Container d = _getFromCachePath(path); + if (null != d) + return d; + + // Special case for ROOT -- we want to throw instead of returning null + if (path.equals(Path.rootPath)) + { + try (DbScope.Transaction t = ensureTransaction()) + { + TableInfo tinfo = CORE.getTableInfoContainers(); + + // Unusual, but possible -- if cache loader hits an exception it can end up caching null + if (null == tinfo) + throw new RootContainerException("Container table could not be retrieved from the cache"); + + // This might be called at bootstrap, before schemas have been created + if (tinfo.getTableType() == DatabaseTableType.NOT_IN_DB) + throw new RootContainerException("Container table has not been created"); + + Container result = new SqlSelector(CORE.getSchema(),"SELECT * FROM " + tinfo + " WHERE Parent IS NULL").getObject(Container.class); + + if (result == null) + throw new RootContainerException("Root container does not exist"); + + _addToCache(result); + // No database changes to commit, but need to decrement the counter + t.commit(); + return result; + } + } + else + { + Path parent = path.getParent(); + String name = path.getName(); + Container dirParent = getForPath(parent); + + if (null == dirParent) + return null; + + Map map = getChildrenMap(dirParent); + return map.get(name); + } + } + + public static class RootContainerException extends RuntimeException + { + private RootContainerException(String message, Throwable cause) + { + super(message, cause); + } + + private RootContainerException(String message) + { + super(message); + } + } + + public static Container getRoot() + { + try + { + return getForPath("/"); + } + catch (MinorConfigurationException e) + { + // If the server is misconfigured, rethrow so some callers don't swallow it and other callers don't end up + // reporting it to mothership, Issue 50843. + throw e; + } + catch (Exception e) + { + // Some callers catch and ignore this exception, e.g., early in the bootstrap process + throw new RootContainerException("Root container can't be retrieved", e); + } + } + + public static void saveAliasesForContainer(Container container, List aliases, User user) + { + Set originalAliases = new CaseInsensitiveHashSet(getAliasesForContainer(container)); + Set newAliases = new CaseInsensitiveHashSet(aliases); + + if (originalAliases.equals(newAliases)) + { + return; + } + + try (DbScope.Transaction transaction = ensureTransaction()) + { + // Delete all of the aliases for the current container, plus any of the aliases that might be associated + // with another container right now + SQLFragment deleteSQL = new SQLFragment(); + deleteSQL.append("DELETE FROM "); + deleteSQL.append(CORE.getTableInfoContainerAliases()); + deleteSQL.append(" WHERE ContainerRowId = ? "); + deleteSQL.add(container.getRowId()); + if (!aliases.isEmpty()) + { + deleteSQL.append(" OR Path IN ("); + String separator = ""; + for (String alias : aliases) + { + deleteSQL.append(separator); + separator = ", "; + deleteSQL.append("LOWER(?)"); + deleteSQL.add(alias); + } + deleteSQL.append(")"); + } + new SqlExecutor(CORE.getSchema()).execute(deleteSQL); + + // Store the alias as LOWER() so that we can query against it using the index + for (String alias : newAliases) + { + SQLFragment insertSQL = new SQLFragment(); + insertSQL.append("INSERT INTO "); + insertSQL.append(CORE.getTableInfoContainerAliases()); + insertSQL.append(" (Path, ContainerRowId) VALUES (LOWER(?), ?)"); + insertSQL.add(alias); + insertSQL.add(container.getRowId()); + new SqlExecutor(CORE.getSchema()).execute(insertSQL); + } + + addAuditEvent(user, container, + "Changed folder aliases from \"" + + StringUtils.join(originalAliases, ", ") + "\" to \"" + + StringUtils.join(newAliases, ", ") + "\""); + + transaction.commit(); + } + } + + // Abstract base class used for attaching system resources (favorite icons, logos, stylesheets, sso auth logos) to folders and projects + public static abstract class ContainerParent implements AttachmentParent + { + private final Container _c; + + protected ContainerParent(Container c) + { + _c = c; + } + + @Override + public String getEntityId() + { + return _c.getId(); + } + + @Override + public String getContainerId() + { + return _c.getId(); + } + + public Container getContainer() + { + return _c; + } + } + + public static Container getHomeContainer() + { + return getForPath(HOME_PROJECT_PATH); + } + + public static List getProjects() + { + return getChildren(getRoot()); + } + + public static NavTree getProjectList(ViewContext context, boolean includeChildren) + { + User user = context.getUser(); + Container currentProject = context.getContainer().getProject(); + String projectNavTreeId = PROJECT_LIST_ID; + if (currentProject != null) + projectNavTreeId += currentProject.getId(); + + NavTree navTree = (NavTree) NavTreeManager.getFromCache(projectNavTreeId, context); + if (null != navTree) + return navTree; + + NavTree list = new NavTree("Projects"); + List projects = getProjects(); + + for (Container project : projects) + { + boolean shouldDisplay = project.shouldDisplay(user) && project.hasPermission("getProjectList()", user, ReadPermission.class); + boolean includeCurrentProject = includeChildren && currentProject != null && currentProject.equals(project); + + if (shouldDisplay || includeCurrentProject) + { + ActionURL startURL = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(project); + + if (includeChildren) + list.addChild(getFolderListForUser(project, context)); + else if (project.equals(getHomeContainer())) + list.addChild(new NavTree("Home", startURL)); + else + list.addChild(project.getTitle(), startURL); + } + } + + list.setId(projectNavTreeId); + NavTreeManager.cacheTree(list, context.getUser()); + + return list; + } + + public static NavTree getFolderListForUser(final Container project, ViewContext viewContext) + { + final boolean isNavAccessOpen = AppProps.getInstance().isNavigationAccessOpen(); + final Container c = viewContext.getContainer(); + final String cacheKey = isNavAccessOpen ? project.getId() : c.getId(); + + NavTree tree = (NavTree) NavTreeManager.getFromCache(cacheKey, viewContext); + if (null != tree) + return tree; + + try + { + assert SecurityLogger.indent("getFolderListForUser()"); + + User user = viewContext.getUser(); + String projectId = project.getId(); + + List folders = new ArrayList<>(getAllChildren(project)); + + Collections.sort(folders); + + Set containersInTree = new HashSet<>(); + + Map m = new HashMap<>(); + Map permission = new HashMap<>(); + + for (Container f : folders) + { + if (!f.isInFolderNav()) + continue; + + boolean hasPolicyRead = f.hasPermission(user, ReadPermission.class); + + boolean skip = ( + !hasPolicyRead || + !f.shouldDisplay(user) || + !f.hasPermission(user, ReadPermission.class) + ); + + //Always put the project and current container in... + if (skip && !f.equals(project) && !f.equals(c)) + continue; + + //HACK to make home link consistent... + String name = f.getTitle(); + if (name.equals("home") && f.equals(getHomeContainer())) + name = "Home"; + + NavTree t = new NavTree(name); + + // 34137: Support folder path expansion for containers where label != name + t.setId(f.getId()); + if (hasPolicyRead) + { + ActionURL url = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(f); + t.setHref(url.getEncodedLocalURIString()); + } + + boolean addFolder = false; + + if (isNavAccessOpen) + { + addFolder = true; + } + else + { + // 32718: If navigation access is not open then hide projects that aren't directly + // accessible in site folder navigation. + + if (f.equals(c) || f.isRoot() || (hasPolicyRead && f.isProject())) + { + // In current container, root, or readable project + addFolder = true; + } + else + { + boolean isAscendant = f.isDescendant(c); + boolean isDescendant = c.isDescendant(f); + boolean inActivePath = isAscendant || isDescendant; + boolean hasAncestryRead = false; + + if (inActivePath) + { + Container leaf = isAscendant ? f : c; + Container localRoot = isAscendant ? c : f; + + List ancestors = containersToRootList(leaf); + Collections.reverse(ancestors); + + for (Container p : ancestors) + { + if (!permission.containsKey(p.getId())) + permission.put(p.getId(), p.hasPermission(user, ReadPermission.class)); + boolean hasRead = permission.get(p.getId()); + + if (p.equals(localRoot)) + { + hasAncestryRead = hasRead; + break; + } + else if (!hasRead) + { + hasAncestryRead = false; + break; + } + } + } + else + { + hasAncestryRead = containersToRoot(f).stream().allMatch(p -> { + if (!permission.containsKey(p.getId())) + permission.put(p.getId(), p.hasPermission(user, ReadPermission.class)); + return permission.get(p.getId()); + }); + } + + if (hasPolicyRead && hasAncestryRead && inActivePath) + { + // Is in the direct readable lineage of the current container + addFolder = true; + } + else if (hasPolicyRead && f.getParent().equals(c.getParent())) + { + // Is a readable sibling of the current container + addFolder = true; + } + else if (hasAncestryRead) + { + // Is a part of a fully readable ancestry + addFolder = true; + } + } + + if (!addFolder) + LOG.debug("isNavAccessOpen restriction: \"" + f.getPath() + "\""); + } + + if (addFolder) + { + containersInTree.add(f); + m.put(f.getId(), t); + } + } + + //Ensure parents of any accessible folder are in the tree. If not add them with no link. + for (Container treeContainer : containersInTree) + { + if (!treeContainer.equals(project) && !containersInTree.contains(treeContainer.getParent())) + { + Set containersToRoot = containersToRoot(treeContainer); + //Possible will be added more than once, if several children are accessible, but that's OK... + for (Container missing : containersToRoot) + { + if (!m.containsKey(missing.getId())) + { + if (isNavAccessOpen) + { + NavTree noLinkTree = new NavTree(missing.getName()); + noLinkTree.setId(missing.getId()); + m.put(missing.getId(), noLinkTree); + } + else + { + if (!permission.containsKey(missing.getId())) + permission.put(missing.getId(), missing.hasPermission(user, ReadPermission.class)); + + if (!permission.get(missing.getId())) + { + NavTree noLinkTree = new NavTree(missing.getName()); + m.put(missing.getId(), noLinkTree); + } + else + { + NavTree linkTree = new NavTree(missing.getName()); + ActionURL url = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(missing); + linkTree.setHref(url.getEncodedLocalURIString()); + m.put(missing.getId(), linkTree); + } + } + } + } + } + } + + for (Container f : folders) + { + if (f.getId().equals(projectId)) + continue; + + NavTree child = m.get(f.getId()); + if (null == child) + continue; + + NavTree parent = m.get(f.getParent().getId()); + assert null != parent; //This should not happen anymore, we assure all parents are in tree. + if (null != parent) + parent.addChild(child); + } + + NavTree projectTree = m.get(projectId); + + projectTree.setId(cacheKey); + + NavTreeManager.cacheTree(projectTree, user); + return projectTree; + } + finally + { + assert SecurityLogger.outdent(); + } + } + + public static Set containersToRoot(Container child) + { + Set containersOnPath = new HashSet<>(); + Container current = child; + while (current != null && !current.isRoot()) + { + containersOnPath.add(current); + current = current.getParent(); + } + + return containersOnPath; + } + + /** + * Provides a sorted list of containers from the root to the child container provided. + * It does not include the root node. + * @param child Container from which the search is sourced. + * @return List sorted in order of distance from root. + */ + public static List containersToRootList(Container child) + { + List containers = new ArrayList<>(); + Container current = child; + while (current != null && !current.isRoot()) + { + containers.add(current); + current = current.getParent(); + } + + Collections.reverse(containers); + return containers; + } + + // Move a container to another part of the container tree. Careful: this method DOES NOT prevent you from orphaning + // an entire tree (e.g., by setting a container's parent to one of its children); the UI in AdminController does this. + // + // NOTE: Beware side-effect of changing ACLs and GROUPS if a container changes projects + // + // @return true if project has changed (should probably redirect to security page) + public static boolean move(Container c, final Container newParent, User user) throws ValidationException + { + if (!isRenameable(c)) + { + throw new IllegalArgumentException("Can't move container " + c.getPath()); + } + + try (QuietCloser ignored = lockForMutation(MutatingOperation.move, c)) + { + List errors = new ArrayList<>(); + for (ContainerListener listener : getListeners()) + { + try + { + errors.addAll(listener.canMove(c, newParent, user)); + } + catch (Exception e) + { + ExceptionUtil.logExceptionToMothership(null, new IllegalStateException(listener.getClass().getName() + ".canMove() threw an exception or violated @NotNull contract")); + } + } + if (!errors.isEmpty()) + { + ValidationException exception = new ValidationException(); + for (String error : errors) + { + exception.addError(new SimpleValidationError(error)); + } + throw exception; + } + + if (c.getParent().getId().equals(newParent.getId())) + return false; + + Container oldParent = c.getParent(); + Container oldProject = c.getProject(); + Container newProject = newParent.isRoot() ? c : newParent.getProject(); + + boolean changedProjects = !oldProject.getId().equals(newProject.getId()); + + // Synchronize the transaction, but not the listeners -- see #9901 + try (DbScope.Transaction t = ensureTransaction()) + { + new SqlExecutor(CORE.getSchema()).execute("UPDATE " + CORE.getTableInfoContainers() + " SET Parent = ? WHERE EntityId = ?", newParent.getId(), c.getId()); + + // Refresh the container directly from the database so the container reflects the new parent, isProject(), etc. + c = getForRowId(c.getRowId()); + + // this could be done in the trigger, but I prefer to put it in the transaction + if (changedProjects) + SecurityManager.changeProject(c, oldProject, newProject, user); + + clearCache(); + + try + { + ExperimentService.get().moveContainer(c, oldParent, newParent); + } + catch (ExperimentException e) + { + throw new RuntimeException(e); + } + + // Clear after the commit has propagated the state to other threads and transactions + // Do this in a commit task in case we've joined another existing DbScope.Transaction instead of starting our own + t.addCommitTask(() -> + { + clearCache(); + getChildrenMap(newParent); // reload the cache + }, DbScope.CommitTaskOption.POSTCOMMIT); + + t.commit(); + } + + Container newContainer = getForId(c.getId()); + fireMoveContainer(newContainer, oldParent, user); + + return changedProjects; + } + } + + public static void rename(@NotNull Container c, User user, String name) + { + rename(c, user, name, c.getTitle(), false); + } + + /** + * Transacted method to rename a container. Optionally, supports updating the title and aliasing the + * original container path when the name is changed (as name changes result in a new container path). + */ + public static Container rename(@NotNull Container c, User user, String name, @Nullable String title, boolean addAlias) + { + try (QuietCloser ignored = lockForMutation(MutatingOperation.rename, c); + DbScope.Transaction tx = ensureTransaction()) + { + final String oldName = c.getName(); + final String newName = StringUtils.trimToNull(name); + boolean isRenaming = !oldName.equals(newName); + StringBuilder errors = new StringBuilder(); + + // Rename + if (isRenaming) + { + // Issue 16221: Don't allow renaming of system reserved folders (e.g. /Shared, home, root, etc). + if (!isRenameable(c)) + throw new ApiUsageException("This folder may not be renamed as it is reserved by the system."); + + if (!Container.isLegalName(newName, c.isProject(), errors)) + throw new ApiUsageException(errors.toString()); + + // Issue 19061: Unable to do case-only container rename + if (c.getParent().hasChild(newName) && !c.equals(c.getParent().getChild(newName))) + { + if (c.getParent().isRoot()) + throw new ApiUsageException("The server already has a project with this name."); + throw new ApiUsageException("The " + (c.getParent().isProject() ? "project " : "folder ") + c.getParent().getPath() + " already has a folder with this name."); + } + + new SqlExecutor(CORE.getSchema()).execute("UPDATE " + CORE.getTableInfoContainers() + " SET Name=? WHERE EntityId=?", newName, c.getId()); + clearCache(); // Clear the entire cache, since containers cache their full paths + // Get new version since name has changed. + Container renamedContainer = getForId(c.getId()); + fireRenameContainer(renamedContainer, user, oldName); + // Clear again after the commit has propagated the state to other threads and transactions + // Do this in a commit task in case we've joined another existing DbScope.Transaction instead of starting our own + tx.addCommitTask(ContainerManager::clearCache, DbScope.CommitTaskOption.POSTCOMMIT); + + // Alias + if (addAlias) + { + // Intentionally use original container rather than the already renamedContainer + List newAliases = new ArrayList<>(getAliasesForContainer(c)); + newAliases.add(c.getPath()); + saveAliasesForContainer(c, newAliases, user); + } + } + + // Title + if (!c.getTitle().equals(title)) + { + if (!Container.isLegalTitle(title, errors)) + throw new ApiUsageException(errors.toString()); + updateTitle(c, title, user); + } + + tx.commit(); + } + catch (ValidationException e) + { + throw new IllegalArgumentException(e); + } + + return getForId(c.getId()); + } + + public static void setChildOrderToAlphabetical(Container parent) + { + setChildOrder(parent.getChildren(), true); + } + + public static void setChildOrder(Container parent, List orderedChildren) throws ContainerException + { + for (Container child : orderedChildren) + { + if (child == null || child.getParent() == null || !child.getParent().equals(parent)) // #13481 + throw new ContainerException("Invalid parent container of " + (child == null ? "null child container" : child.getPath())); + } + setChildOrder(orderedChildren, false); + } + + private static void setChildOrder(List siblings, boolean resetToAlphabetical) + { + try (DbScope.Transaction t = ensureTransaction()) + { + for (int index = 0; index < siblings.size(); index++) + { + Container current = siblings.get(index); + new SqlExecutor(CORE.getSchema()).execute("UPDATE " + CORE.getTableInfoContainers() + " SET SortOrder = ? WHERE EntityId = ?", + resetToAlphabetical ? 0 : index, current.getId()); + } + // Clear after the commit has propagated the state to other threads and transactions + // Do this in a commit task in case we've joined another existing DbScope.Transaction instead of starting our own + t.addCommitTask(ContainerManager::clearCache, DbScope.CommitTaskOption.POSTCOMMIT); + + t.commit(); + } + } + + private enum MutatingOperation + { + delete, + rename, + move + } + + private static final Map mutatingContainers = Collections.synchronizedMap(new IntHashMap<>()); + + private static QuietCloser lockForMutation(MutatingOperation op, Container c) + { + return lockForMutation(op, Collections.singletonList(c)); + } + + private static QuietCloser lockForMutation(MutatingOperation op, Collection containers) + { + List ids = new ArrayList<>(containers.size()); + synchronized (mutatingContainers) + { + for (Container container : containers) + { + MutatingOperation currentOp = mutatingContainers.get(container.getRowId()); + if (currentOp != null) + { + throw new ApiUsageException("Cannot start a " + op + " operation on " + container.getPath() + ". It is currently undergoing a " + currentOp); + } + ids.add(container.getRowId()); + } + ids.forEach(id -> mutatingContainers.put(id, op)); + } + return () -> + { + synchronized (mutatingContainers) + { + ids.forEach(mutatingContainers::remove); + } + }; + } + + // Delete containers from the database + private static boolean delete(final Collection containers, User user, @Nullable String comment) + { + // Do this check before we bother with any synchronization + for (Container container : containers) + { + if (!isDeletable(container)) + { + throw new ApiUsageException("Cannot delete container: " + container.getPath()); + } + } + + try (QuietCloser ignored = lockForMutation(MutatingOperation.delete, containers)) + { + boolean deleted = true; + for (Container c : containers) + { + deleted = deleted && delete(c, user, comment); + } + return deleted; + } + } + + // Delete a container from the database + private static boolean delete(final Container c, User user, @Nullable String comment) + { + // Verify method isn't called inappropriately + if (mutatingContainers.get(c.getRowId()) != MutatingOperation.delete) + { + throw new IllegalStateException("Container not flagged as being deleted: " + c.getPath()); + } + + LOG.debug("Starting container delete for " + c.getContainerNoun(true) + " " + c.getPath()); + + // Tell the search indexer to drop work for the container that's about to be deleted + SearchService.get().purgeForContainer(c); + + DbScope.RetryFn tryDeleteContainer = (tx) -> + { + // Verify that no children exist + Selector sel = new TableSelector(CORE.getTableInfoContainers(), new SimpleFilter(FieldKey.fromParts("Parent"), c), null); + + if (sel.exists()) + { + _removeFromCache(c, true); + return false; + } + + if (c.shouldRemoveFromPortal()) + { + // Need to remove portal page, too; container name is page's pageId and in container's parent container + Portal.PortalPage page = Portal.getPortalPage(c.getParent(), c.getName()); + if (null != page) // Be safe + Portal.deletePage(page); + + // Tell parent + setContainerTabDeleted(c.getParent(), c.getName(), c.getParent().getFolderType().getName()); + } + + fireDeleteContainer(c, user); + + SqlExecutor sqlExecutor = new SqlExecutor(CORE.getSchema()); + sqlExecutor.execute("DELETE FROM " + CORE.getTableInfoContainerAliases() + " WHERE ContainerRowId=?", c.getRowId()); + sqlExecutor.execute("DELETE FROM " + CORE.getTableInfoContainers() + " WHERE EntityId=?", c.getId()); + // now that the container is actually gone, delete all ACLs (better to have an ACL w/o object than object w/o ACL) + SecurityPolicyManager.removeAll(c); + // and delete all container-based sequences + DbSequenceManager.deleteAll(c); + + ExperimentService experimentService = ExperimentService.get(); + if (experimentService != null) + experimentService.removeContainerDataTypeExclusions(c.getId()); + + // After we've committed the transaction, be sure that we remove this container from the cache + // See https://www.labkey.org/issues/home/Developer/issues/details.view?issueId=17015 + tx.addCommitTask(() -> + { + // Be sure that we've waited until any threads that might be populating the cache have finished + // before we guarantee that we've removed this now-deleted container + DATABASE_QUERY_LOCK.lock(); + try + { + _removeFromCache(c, true); + } + finally + { + DATABASE_QUERY_LOCK.unlock(); + } + }, DbScope.CommitTaskOption.POSTCOMMIT); + String auditComment = c.getContainerNoun(true) + " " + c.getPath() + " was deleted"; + if (comment != null) + auditComment = auditComment.concat(". " + comment); + addAuditEvent(user, c, auditComment); + return true; + }; + + boolean success = CORE.getSchema().getScope().executeWithRetry(tryDeleteContainer); + if (success) + { + LOG.debug("Completed container delete for " + c.getContainerNoun(true) + " " + c.getPath()); + } + else + { + LOG.warn("Failed to delete container: " + c.getPath()); + } + return success; + } + + /** + * Delete a single container. Primarily for use by tests. + */ + public static boolean delete(final Container c, User user) + { + return delete(List.of(c), user, null); + } + + public static boolean isDeletable(Container c) + { + return !isSystemContainer(c); + } + + public static boolean isRenameable(Container c) + { + return !isSystemContainer(c); + } + + /** System containers include the root container, /Home, and /Shared */ + public static boolean isSystemContainer(Container c) + { + return c.equals(getRoot()) || c.equals(getHomeContainer()) || c.equals(getSharedContainer()); + } + + /** Has the container already been deleted or is it in the process of being deleted? */ + public static boolean exists(@Nullable Container c) + { + return c != null && null != getForId(c.getEntityId()) && mutatingContainers.get(c.getRowId()) != MutatingOperation.delete; + } + + public static void deleteAll(Container root, User user, @Nullable String comment) throws UnauthorizedException + { + if (!hasTreePermission(root, user, DeletePermission.class)) + throw new UnauthorizedException("You don't have delete permissions to all folders"); + + LOG.debug("Starting container (and children) delete for " + root.getContainerNoun(true) + " " + root.getPath()); + Set depthFirst = getAllChildrenDepthFirst(root); + depthFirst.add(root); + + delete(depthFirst, user, comment); + + LOG.debug("Completed container (and children) delete for " + root.getContainerNoun(true) + " " + root.getPath()); + } + + public static void deleteAll(Container root, User user) throws UnauthorizedException + { + deleteAll(root, user, null); + } + + private static void addAuditEvent(User user, Container c, String comment) + { + if (user != null) + { + AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, c, comment); + AuditLogService.get().addEvent(user, event); + } + } + + private static Set getAllChildrenDepthFirst(Container c) + { + Set set = new LinkedHashSet<>(); + getAllChildrenDepthFirst(c, set); + return set; + } + + private static void getAllChildrenDepthFirst(Container c, Collection list) + { + for (Container child : c.getChildren()) + { + getAllChildrenDepthFirst(child, list); + list.add(child); + } + } + + private static Container _getFromCachePath(Path path) + { + return CACHE_PATH.get(path); + } + + private static Container _addToCache(Container c) + { + assert DATABASE_QUERY_LOCK.isHeldByCurrentThread() : "Any cache modifications must be synchronized at a " + + "higher level so that we ensure that the container to be inserted still exists and hasn't been deleted"; + CACHE_ENTITY_ID.put(c.getEntityId(), c); + CACHE_PATH.put(c.getParsedPath(), c); + return c; + } + + private static void _clearChildrenFromCache(Container c) + { + CACHE_CHILDREN.remove(c.getEntityId()); + navTreeManageUncache(c); + } + + /** @param hierarchyChange whether the shape of the container tree has changed */ + private static void _removeFromCache(Container c, boolean hierarchyChange) + { + CACHE_ENTITY_ID.remove(c.getEntityId()); + CACHE_PATH.remove(c.getParsedPath()); + + if (hierarchyChange) + { + // This is strictly keeping track of the parent/child relationships themselves so it only needs to be + // cleared when the tree changes + CACHE_CHILDREN.clear(); + } + + navTreeManageUncache(c); + } + + public static void clearCache() + { + CACHE_PATH.clear(); + CACHE_ENTITY_ID.clear(); + CACHE_CHILDREN.clear(); + + // UNDONE: NavTreeManager should register a ContainerListener + NavTreeManager.uncacheAll(); + } + + private static void navTreeManageUncache(Container c) + { + // UNDONE: NavTreeManager should register a ContainerListener + NavTreeManager.uncacheTree(PROJECT_LIST_ID); + NavTreeManager.uncacheTree(getRoot().getId()); + + Container project = c.getProject(); + if (project != null) + { + NavTreeManager.uncacheTree(project.getId()); + NavTreeManager.uncacheTree(PROJECT_LIST_ID + project.getId()); + } + } + + public static void notifyContainerChange(String id, Property prop) + { + notifyContainerChange(id, prop, null); + } + + public static void notifyContainerChange(String id, Property prop, @Nullable User u) + { + if (_constructing.contains(new GUID(id))) + return; + + Container c = getForId(id); + if (null != c) + { + _removeFromCache(c, false); + c = getForId(id); // load a fresh container since the original might be stale. + if (null != c) + { + ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(c, u, prop, null, null); + firePropertyChangeEvent(evt); + } + } + } + + + /** Recursive, including root node */ + public static Set getAllChildren(Container root) + { + Set children = getAllChildrenDepthFirst(root); + children.add(root); + + return Collections.unmodifiableSet(children); + } + + /** + * Return all children of the root node, including root node, which have the given active module + */ + @NotNull + public static Set getAllChildrenWithModule(@NotNull Container root, @NotNull Module module) + { + Set children = new HashSet<>(); + for (Container candidate : getAllChildren(root)) + { + if (candidate.getActiveModules().contains(module)) + children.add(candidate); + } + return Collections.unmodifiableSet(children); + } + + public static long getContainerCount() + { + return new TableSelector(CORE.getTableInfoContainers()).getRowCount(); + } + + public static long getWorkbookCount() + { + return new TableSelector(CORE.getTableInfoContainers(), new SimpleFilter(FieldKey.fromParts("type"), "workbook"), null).getRowCount(); + } + + public static long getArchivedContainerCount() + { + return new TableSelector(CORE.getTableInfoContainers(), new SimpleFilter(FieldKey.fromParts("lockstate"), "Archived"), null).getRowCount(); + } + + public static long getAuditCommentRequiredCount() + { + SQLFragment sql = new SQLFragment( + "SELECT COUNT(*) FROM\n" + + " core.containers c\n" + + " JOIN prop.propertysets ps on c.entityid = ps.objectid\n" + + " JOIN prop.properties p on p.\"set\" = ps.\"set\"\n" + + "WHERE ps.category = '" + AUDIT_SETTINGS_PROPERTY_SET_NAME + "' AND p.name='"+ REQUIRE_USER_COMMENTS_PROPERTY_NAME + "' and p.value='true'"); + return new SqlSelector(CORE.getSchema(), sql).getObject(Long.class); + } + + + /** Retrieve entire container hierarchy */ + public static MultiValuedMap getContainerTree() + { + final MultiValuedMap mm = new ArrayListValuedHashMap<>(); + + // Get all containers and parents + SqlSelector selector = new SqlSelector(CORE.getSchema(), "SELECT Parent, EntityId FROM " + CORE.getTableInfoContainers() + " ORDER BY SortOrder, LOWER(Name) ASC"); + + selector.forEach(rs -> { + String parentId = rs.getString(1); + Container parent = (parentId != null ? getForId(parentId) : null); + Container child = getForId(rs.getString(2)); + + if (null != child) + mm.put(parent, child); + }); + + return mm; + } + + /** + * Returns a branch of the container tree including only the root and its descendants + * @param root The root container + * @return MultiMap of containers including root and its descendants + */ + public static MultiValuedMap getContainerTree(Container root) + { + //build a multimap of only the container ids + final MultiValuedMap mmIds = new ArrayListValuedHashMap<>(); + + // Get all containers and parents + Selector selector = new SqlSelector(CORE.getSchema(), "SELECT Parent, EntityId FROM " + CORE.getTableInfoContainers() + " ORDER BY SortOrder, LOWER(Name) ASC"); + + selector.forEach(rs -> mmIds.put(rs.getString(1), rs.getString(2))); + + //now find the root and build a MultiMap of it and its descendants + MultiValuedMap mm = new ArrayListValuedHashMap<>(); + mm.put(null, root); + addChildren(root, mmIds, mm); + return mm; + } + + private static void addChildren(Container c, MultiValuedMap mmIds, MultiValuedMap mm) + { + Collection childIds = mmIds.get(c.getId()); + if (null != childIds) + { + for (String childId : childIds) + { + Container child = getForId(childId); + if (null != child) + { + mm.put(c, child); + addChildren(child, mmIds, mm); + } + } + } + } + + public static Set getContainerSet(MultiValuedMap mm, User user, Class perm) + { + Collection containers = mm.values(); + if (null == containers) + return new HashSet<>(); + + return containers + .stream() + .filter(c -> c.hasPermission(user, perm)) + .collect(Collectors.toSet()); + } + + + public static SQLFragment getIdsAsCsvList(Set containers, SqlDialect d) + { + if (containers.isEmpty()) + return new SQLFragment("(NULL)"); // WHERE x IN (NULL) should match no rows + + SQLFragment csvList = new SQLFragment("("); + String comma = ""; + for (Container container : containers) + { + csvList.append(comma); + comma = ","; + csvList.appendValue(container, d); + } + csvList.append(")"); + + return csvList; + } + + + public static List getIds(User user, Class perm) + { + Set containers = getContainerSet(getContainerTree(), user, perm); + + List ids = new ArrayList<>(containers.size()); + + for (Container c : containers) + ids.add(c.getId()); + + return ids; + } + + + // + // ContainerListener + // + + public interface ContainerListener extends PropertyChangeListener + { + enum Order {First, Last} + + /** Called after a new container has been created */ + void containerCreated(Container c, User user); + + default void containerCreated(Container c, User user, @Nullable String auditMsg) + { + containerCreated(c, user); + } + + /** Called immediately prior to deleting the row from core.containers */ + void containerDeleted(Container c, User user); + + /** Called after the container has been moved to its new parent */ + void containerMoved(Container c, Container oldParent, User user); + + /** + * Called prior to moving a container, to find out if there are any issues that would prevent a successful move + * @return a list of errors that should prevent the move from happening, if any + */ + @NotNull + Collection canMove(Container c, Container newParent, User user); + + @Override + void propertyChange(PropertyChangeEvent evt); + } + + public static abstract class AbstractContainerListener implements ContainerListener + { + @Override + public void containerCreated(Container c, User user) + {} + + @Override + public void containerDeleted(Container c, User user) + {} + + @Override + public void containerMoved(Container c, Container oldParent, User user) + {} + + @NotNull + @Override + public Collection canMove(Container c, Container newParent, User user) + { + return Collections.emptyList(); + } + + @Override + public void propertyChange(PropertyChangeEvent evt) + {} + } + + + public static class ContainerPropertyChangeEvent extends PropertyChangeEvent implements PropertyChange + { + public final Property property; + public final Container container; + public User user; + + public ContainerPropertyChangeEvent(Container c, @Nullable User user, Property p, Object oldValue, Object newValue) + { + super(c, p.name(), oldValue, newValue); + container = c; + this.user = user; + property = p; + } + + public ContainerPropertyChangeEvent(Container c, Property p, Object oldValue, Object newValue) + { + this(c, null, p, oldValue, newValue); + } + + @Override + public Property getProperty() + { + return property; + } + } + + + // Thread-safe list implementation that allows iteration and modifications without external synchronization + private static final List _listeners = new CopyOnWriteArrayList<>(); + private static final List _laterListeners = new CopyOnWriteArrayList<>(); + + // These listeners are executed in the order they are registered, before the "Last" listeners + public static void addContainerListener(ContainerListener listener) + { + addContainerListener(listener, ContainerListener.Order.First); + } + + + // Explicitly request "Last" ordering via this method. "Last" listeners execute after all "First" listeners. + public static void addContainerListener(ContainerListener listener, ContainerListener.Order order) + { + if (ContainerListener.Order.First == order) + _listeners.add(listener); + else + _laterListeners.add(listener); + } + + + public static void removeContainerListener(ContainerListener listener) + { + _listeners.remove(listener); + _laterListeners.remove(listener); + } + + + private static List getListeners() + { + List combined = new ArrayList<>(_listeners.size() + _laterListeners.size()); + combined.addAll(_listeners); + combined.addAll(_laterListeners); + + return combined; + } + + + private static List getListenersReversed() + { + List combined = new LinkedList<>(); + + // Copy to guarantee consistency between .listIterator() and .size() + List copy = new ArrayList<>(_listeners); + ListIterator iter = copy.listIterator(copy.size()); + + // Iterate in reverse + while(iter.hasPrevious()) + combined.add(iter.previous()); + + // Copy to guarantee consistency between .listIterator() and .size() + // Add elements from the laterList in reverse order so that Core is fired last + List laterCopy = new ArrayList<>(_laterListeners); + ListIterator laterIter = laterCopy.listIterator(laterCopy.size()); + + // Iterate in reverse + while(laterIter.hasPrevious()) + combined.add(laterIter.previous()); + + return combined; + } + + + protected static void fireCreateContainer(Container c, User user, @Nullable String auditMsg) + { + List list = getListeners(); + + for (ContainerListener cl : list) + { + try + { + cl.containerCreated(c, user, auditMsg); + } + catch (Throwable t) + { + LOG.error("fireCreateContainer for " + cl.getClass().getName(), t); + } + } + } + + + protected static void fireDeleteContainer(Container c, User user) + { + List list = getListenersReversed(); + + for (ContainerListener l : list) + { + LOG.debug("Deleting " + c.getPath() + ": fireDeleteContainer for " + l.getClass().getName()); + try + { + l.containerDeleted(c, user); + } + catch (RuntimeException e) + { + LOG.error("fireDeleteContainer for " + l.getClass().getName(), e); + + // Fail fast (first Throwable aborts iteration), #17560 + throw e; + } + } + } + + + protected static void fireRenameContainer(Container c, User user, String oldValue) + { + ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(c, user, Property.Name, oldValue, c.getName()); + firePropertyChangeEvent(evt); + } + + + protected static void fireMoveContainer(Container c, Container oldParent, User user) + { + List list = getListeners(); + + for (ContainerListener cl : list) + { + // While we would ideally transact the full container move, that will likely cause long-blocking + // queries and/or deadlocks. For now, at least transact each separate move handler independently + try (DbScope.Transaction transaction = CoreSchema.getInstance().getSchema().getScope().ensureTransaction()) + { + cl.containerMoved(c, oldParent, user); + transaction.commit(); + } + } + ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(c, user, Property.Parent, oldParent, c.getParent()); + firePropertyChangeEvent(evt); + } + + + public static void firePropertyChangeEvent(ContainerPropertyChangeEvent evt) + { + if (_constructing.contains(evt.container.getEntityId())) + return; + + List list = getListeners(); + for (ContainerListener l : list) + { + try + { + l.propertyChange(evt); + } + catch (Throwable t) + { + LOG.error("firePropertyChangeEvent for " + l.getClass().getName(), t); + } + } + } + + private static final List MODULE_DEPENDENCY_PROVIDERS = new CopyOnWriteArrayList<>(); + + public static void registerModuleDependencyProvider(ModuleDependencyProvider provider) + { + MODULE_DEPENDENCY_PROVIDERS.add(provider); + } + + public static void forEachModuleDependencyProvider(Consumer action) + { + MODULE_DEPENDENCY_PROVIDERS.forEach(action); + } + + // Compliance module adds a locked project handler that checks permissions; without that, this implementation + // is used, and projects are never locked + static volatile LockedProjectHandler LOCKED_PROJECT_HANDLER = (project, user, contextualRoles, lockState) -> false; + + // Replaces any previously set LockedProjectHandler + public static void setLockedProjectHandler(LockedProjectHandler handler) + { + LOCKED_PROJECT_HANDLER = handler; + } + + public static Container createDefaultSupportContainer() + { + LOG.info("Creating default support container: " + DEFAULT_SUPPORT_PROJECT_PATH); + // create a "support" container. Admins can do anything, + // Users can read/write, Guests can read. + return bootstrapContainer(DEFAULT_SUPPORT_PROJECT_PATH, + RoleManager.getRole(AuthorRole.class), + RoleManager.getRole(ReaderRole.class) + ); + } + + public static void removeDefaultSupportContainer(User user) + { + Container support = getDefaultSupportContainer(); + if (support != null) + { + LOG.info("Removing default support container: " + DEFAULT_SUPPORT_PROJECT_PATH); + ContainerManager.delete(support, user); + } + } + + public static Container getDefaultSupportContainer() + { + return getForPath(DEFAULT_SUPPORT_PROJECT_PATH); + } + + public static List getAliasesForContainer(Container c) + { + return Collections.unmodifiableList(new SqlSelector(CORE.getSchema(), + new SQLFragment("SELECT Path FROM " + CORE.getTableInfoContainerAliases() + " WHERE ContainerRowId = ? ORDER BY Path", + c.getRowId())).getArrayList(String.class)); + } + + @Nullable + public static Container resolveContainerPathAlias(String path) + { + return resolveContainerPathAlias(path, false); + } + + @Nullable + private static Container resolveContainerPathAlias(String path, boolean top) + { + // Strip any trailing slashes + while (path.endsWith("/")) + { + path = path.substring(0, path.length() - 1); + } + + // Simple case -- resolve directly (sans alias) + Container aliased = getForPath(path); + if (aliased != null) + return aliased; + + // Simple case -- directly resolve from database + aliased = getForPathAlias(path); + if (aliased != null) + return aliased; + + // At the leaf and the container was not found + if (top) + return null; + + List splits = Arrays.asList(path.split("/")); + String subPath = ""; + for (int i=0; i < splits.size()-1; i++) // minus 1 due to leaving off last container + { + if (!splits.get(i).isEmpty()) + subPath += "/" + splits.get(i); + } + + aliased = resolveContainerPathAlias(subPath, false); + + if (aliased == null) + return null; + + String leafPath = aliased.getPath() + "/" + splits.get(splits.size()-1); + return resolveContainerPathAlias(leafPath, true); + } + + @Nullable + private static Container getForPathAlias(String path) + { + // We store the path as lower-case, so we don't need to also LOWER() on the value in core.ContainerAliases, letting the DB use the index + Container[] ret = new SqlSelector(CORE.getSchema(), + "SELECT * FROM " + CORE.getTableInfoContainers() + " c, " + CORE.getTableInfoContainerAliases() + " ca WHERE ca.ContainerRowId = c.RowId AND ca.path = LOWER(?)", + path).getArray(Container.class); + + return ret.length == 0 ? null : ret[0]; + } + + public static Container getMoveTargetContainer(@Nullable String queryName, @NotNull Container sourceContainer, User user, @Nullable String targetIdOrPath, Errors errors) + { + if (targetIdOrPath == null) + { + errors.reject(ERROR_GENERIC, "A target container must be specified for the move operation."); + return null; + } + + Container _targetContainer = getContainerForIdOrPath(targetIdOrPath); + if (_targetContainer == null) + { + errors.reject(ERROR_GENERIC, "The target container was not found: " + targetIdOrPath + "."); + return null; + } + + if (!_targetContainer.hasPermission(user, InsertPermission.class)) + { + String _queryName = queryName == null ? "this table" : "'" + queryName + "'"; + errors.reject(ERROR_GENERIC, "You do not have permission to move rows from " + _queryName + " to the target container: " + targetIdOrPath + "."); + return null; + } + + if (!isValidTargetContainer(sourceContainer, _targetContainer)) + { + errors.reject(ERROR_GENERIC, "Invalid target container for the move operation: " + targetIdOrPath + "."); + return null; + } + return _targetContainer; + } + + private static Container getContainerForIdOrPath(String targetContainer) + { + Container c = ContainerManager.getForId(targetContainer); + if (c == null) + c = ContainerManager.getForPath(targetContainer); + + return c; + } + + // targetContainer must be in the same app project at this time + // i.e. child of current project, project of current child, sibling within project + private static boolean isValidTargetContainer(Container current, Container target) + { + if (current.isRoot() || target.isRoot()) + return false; + + // Allow moving to the current container since we now allow the chosen entities to be from different containers + if (current.equals(target)) + return true; + + boolean moveFromProjectToChild = current.isProject() && target.getParent().equals(current); + boolean moveFromChildToProject = !current.isProject() && current.getParent().isProject() && current.getParent().equals(target); + boolean moveFromChildToSibling = !current.isProject() && current.getParent().isProject() && current.getParent().equals(target.getParent()); + + return moveFromProjectToChild || moveFromChildToProject || moveFromChildToSibling; + } + + /** + * Updates the container of specified rows in the provided database table. Optionally, the modification timestamp + * and the user who made the modification can also be updated if specified. + * + * @param dataTable The table where the container update should be applied. + * @param idField The name of the identifier field used to locate the rows to update. + * @param ids A collection of identifier values specifying the rows to be updated. + * @param targetContainer The target container to set for the specified rows. + * @param user The user performing the update. If null, modified/modifiedBy details are not updated. + * @param withModified If true, updates the modified timestamp and the user who made the modification. + * @return The number of rows updated in the table. + */ + public static int updateContainer(TableInfo dataTable, String idField, Collection ids, Container targetContainer, @Nullable User user, boolean withModified) + { + try (DbScope.Transaction transaction = dataTable.getSchema().getScope().ensureTransaction()) + { + SQLFragment dataUpdate = new SQLFragment("UPDATE ").append(dataTable) + .append(" SET container = ").appendValue(targetContainer); + if (withModified) + { + assert user != null : "User must be specified when updating modified/modifiedBy details."; + dataUpdate.append(", modified = ").appendValue(new Date()); + dataUpdate.append(", modifiedby = ").appendValue(user.getUserId()); + } + dataUpdate.append(" WHERE ").appendIdentifier(idField); + dataTable.getSchema().getSqlDialect().appendInClauseSql(dataUpdate, ids); + int numUpdated = new SqlExecutor(dataTable.getSchema()).execute(dataUpdate); + transaction.commit(); + + return numUpdated; + } + } + + /** + * If a container at the given path does not exist, create one and set permissions. If the container does exist, + * permissions are only set if there is no explicit ACL for the container. This prevents us from resetting + * permissions if all users are dropped. Implicitly done as an admin-level service user. + */ + @NotNull + public static Container bootstrapContainer(String path, @NotNull Role userRole, @Nullable Role guestRole) + { + Container c = null; + User user = User.getAdminServiceUser(); + + try + { + c = getForPath(path); + } + catch (RootContainerException e) + { + // Ignore this -- root doesn't exist yet + } + boolean newContainer = false; + + if (c == null) + { + LOG.debug("Creating new container for path '" + path + "'"); + newContainer = true; + c = ensureContainer(path, user); + } + + // Only set permissions if there are no explicit permissions + // set for this object or we just created it + Integer policyCount = null; + if (!newContainer) + { + policyCount = new SqlSelector(CORE.getSchema(), + "SELECT COUNT(*) FROM " + CORE.getTableInfoPolicies() + " WHERE ResourceId = ?", + c.getId()).getObject(Integer.class); + } + + if (newContainer || 0 == policyCount.intValue()) + { + LOG.debug("Setting permissions for '" + path + "'"); + MutableSecurityPolicy policy = new MutableSecurityPolicy(c); + policy.addRoleAssignment(SecurityManager.getGroup(Group.groupUsers), userRole); + if (guestRole != null) + policy.addRoleAssignment(SecurityManager.getGroup(Group.groupGuests), guestRole); + SecurityPolicyManager.savePolicy(policy, user); + } + + return c; + } + + /** + * @param container the container being created. May be null if we haven't actually created it yet + * @param parent the parent of the container being created. Used in case the container doesn't actually exist yet. + * @return the list of standard steps and any extra ones based on the container's FolderType + */ + public static List getCreateContainerWizardSteps(@Nullable Container container, @NotNull Container parent) + { + List navTrail = new ArrayList<>(); + + boolean isProject = parent.isRoot(); + + navTrail.add(new NavTree(isProject ? "Create Project" : "Create Folder")); + navTrail.add(new NavTree("Users / Permissions")); + if (isProject) + navTrail.add(new NavTree("Project Settings")); + if (container != null) + navTrail.addAll(container.getFolderType().getExtraSetupSteps(container)); + return navTrail; + } + + @TestTimeout(120) @TestWhen(TestWhen.When.BVT) + public static class TestCase extends Assert implements ContainerListener + { + Map _containers = new HashMap<>(); + Container _testRoot = null; + + @Before + public void setUp() + { + if (null == _testRoot) + { + Container junit = JunitUtil.getTestContainer(); + _testRoot = ensureContainer(junit, "ContainerManager$TestCase-" + GUID.makeGUID(), TestContext.get().getUser()); + addContainerListener(this); + } + } + + @After + public void tearDown() + { + removeContainerListener(this); + if (null != _testRoot) + deleteAll(_testRoot, TestContext.get().getUser()); + } + + @Test + public void testImproperFolderNamesBlocked() + { + String[] badNames = {"", "f\\o", "f/o", "f\\\\o", "foo;", "@foo", "foo" + '\u001F', '\u0000' + "foo", "fo" + '\u007F' + "o", "" + '\u009F'}; + + for (String name: badNames) + { + try + { + Container c = createContainer(_testRoot, name, TestContext.get().getUser()); + try + { + assertTrue(delete(c, TestContext.get().getUser())); + } + catch (Exception ignored) {} + fail("Should have thrown exception when trying to create container with name: " + name); + } + catch (ApiUsageException e) + { + // Do nothing, this is expected + } + } + } + + @Test + public void testCreateDeleteContainers() + { + int count = 20; + Random random = new Random(); + MultiValuedMap mm = new ArrayListValuedHashMap<>(); + + for (int i = 1; i <= count; i++) + { + int parentId = random.nextInt(i); + String parentName = 0 == parentId ? _testRoot.getName() : String.valueOf(parentId); + String childName = String.valueOf(i); + mm.put(parentName, childName); + } + + logNode(mm, _testRoot.getName(), 0); + for (int i=0; i<2; i++) //do this twice to make sure the containers were *really* deleted + { + createContainers(mm, _testRoot.getName(), _testRoot); + assertEquals(count, _containers.size()); + cleanUpChildren(mm, _testRoot.getName(), _testRoot); + assertEquals(0, _containers.size()); + } + } + + @Test + public void testCache() + { + assertEquals(0, _containers.size()); + assertEquals(0, getChildren(_testRoot).size()); + + Container one = createContainer(_testRoot, "one", TestContext.get().getUser()); + assertEquals(1, _containers.size()); + assertEquals(1, getChildren(_testRoot).size()); + assertEquals(0, getChildren(one).size()); + + Container oneA = createContainer(one, "A", TestContext.get().getUser()); + assertEquals(2, _containers.size()); + assertEquals(1, getChildren(_testRoot).size()); + assertEquals(1, getChildren(one).size()); + assertEquals(0, getChildren(oneA).size()); + + Container oneB = createContainer(one, "B", TestContext.get().getUser()); + assertEquals(3, _containers.size()); + assertEquals(1, getChildren(_testRoot).size()); + assertEquals(2, getChildren(one).size()); + assertEquals(0, getChildren(oneB).size()); + + Container deleteme = createContainer(one, "deleteme", TestContext.get().getUser()); + assertEquals(4, _containers.size()); + assertEquals(1, getChildren(_testRoot).size()); + assertEquals(3, getChildren(one).size()); + assertEquals(0, getChildren(deleteme).size()); + + assertTrue(delete(deleteme, TestContext.get().getUser())); + assertEquals(3, _containers.size()); + assertEquals(1, getChildren(_testRoot).size()); + assertEquals(2, getChildren(one).size()); + + Container oneC = createContainer(one, "C", TestContext.get().getUser()); + assertEquals(4, _containers.size()); + assertEquals(1, getChildren(_testRoot).size()); + assertEquals(3, getChildren(one).size()); + assertEquals(0, getChildren(oneC).size()); + + assertTrue(delete(oneC, TestContext.get().getUser())); + assertTrue(delete(oneB, TestContext.get().getUser())); + assertEquals(1, getChildren(one).size()); + + assertTrue(delete(oneA, TestContext.get().getUser())); + assertEquals(0, getChildren(one).size()); + + assertTrue(delete(one, TestContext.get().getUser())); + assertEquals(0, getChildren(_testRoot).size()); + assertEquals(0, _containers.size()); + } + + @Test + public void testFolderType() + { + // Test all folder types + List folderTypes = new ArrayList<>(FolderTypeManager.get().getAllFolderTypes()); + for (FolderType folderType : folderTypes) + { + if (!folderType.isProjectOnlyType()) // Dataspace can't be subfolder + testOneFolderType(folderType); + } + } + + private void testOneFolderType(FolderType folderType) + { + LOG.info("testOneFolderType(" + folderType.getName() + "): creating container"); + Container newFolder = createContainer(_testRoot, "folderTypeTest", TestContext.get().getUser()); + FolderType ft = newFolder.getFolderType(); + assertEquals(FolderType.NONE, ft); + + Container newFolderFromCache = getForId(newFolder.getId()); + assertNotNull(newFolderFromCache); + assertEquals(FolderType.NONE, newFolderFromCache.getFolderType()); + LOG.info("testOneFolderType(" + folderType.getName() + "): setting folder type"); + newFolder.setFolderType(folderType, TestContext.get().getUser()); + + newFolderFromCache = getForId(newFolder.getId()); + assertNotNull(newFolderFromCache); + assertEquals(newFolderFromCache.getFolderType().getName(), folderType.getName()); + assertEquals(newFolderFromCache.getFolderType().getDescription(), folderType.getDescription()); + + LOG.info("testOneFolderType(" + folderType.getName() + "): deleteAll"); + deleteAll(newFolder, TestContext.get().getUser()); // There might be subfolders because of container tabs + LOG.info("testOneFolderType(" + folderType.getName() + "): deleteAll complete"); + Container deletedContainer = getForId(newFolder.getId()); + + if (deletedContainer != null) + { + fail("Expected container with Id " + newFolder.getId() + " to be deleted, but found " + deletedContainer + ". Folder type was " + folderType); + } + } + + private static void createContainers(MultiValuedMap mm, String name, Container parent) + { + Collection nodes = mm.get(name); + + if (null == nodes) + return; + + for (String childName : nodes) + { + Container child = createContainer(parent, childName, TestContext.get().getUser()); + createContainers(mm, childName, child); + } + } + + private static void cleanUpChildren(MultiValuedMap mm, String name, Container parent) + { + Collection nodes = mm.get(name); + + if (null == nodes) + return; + + for (String childName : nodes) + { + Container child = getForPath(makePath(parent, childName)); + cleanUpChildren(mm, childName, child); + assertTrue(delete(child, TestContext.get().getUser())); + } + } + + private static void logNode(MultiValuedMap mm, String name, int offset) + { + Collection nodes = mm.get(name); + + if (null == nodes) + return; + + for (String childName : nodes) + { + LOG.debug(StringUtils.repeat(" ", offset) + childName); + logNode(mm, childName, offset + 1); + } + } + + // ContainerListener + @Override + public void propertyChange(PropertyChangeEvent evt) + { + } + + @Override + public void containerCreated(Container c, User user) + { + if (null == _testRoot || !c.getParsedPath().startsWith(_testRoot.getParsedPath())) + return; + _containers.put(c.getParsedPath(), c); + } + + + @Override + public void containerDeleted(Container c, User user) + { + _containers.remove(c.getParsedPath()); + } + + @Override + public void containerMoved(Container c, Container oldParent, User user) + { + } + + @NotNull + @Override + public Collection canMove(Container c, Container newParent, User user) + { + return Collections.emptyList(); + } + } + + static + { + ObjectFactory.Registry.register(Container.class, new ContainerFactory()); + } + + public static class ContainerFactory implements ObjectFactory + { + @Override + public Container fromMap(Map m) + { + throw new UnsupportedOperationException(); + } + + @Override + public Container fromMap(Container bean, Map m) + { + throw new UnsupportedOperationException(); + } + + @Override + public Map toMap(Container bean, Map m) + { + throw new UnsupportedOperationException(); + } + + @Override + public Container handle(ResultSet rs) throws SQLException + { + String id; + Container d; + String parentId = rs.getString("Parent"); + String name = rs.getString("Name"); + id = rs.getString("EntityId"); + int rowId = rs.getInt("RowId"); + int sortOrder = rs.getInt("SortOrder"); + Date created = rs.getTimestamp("Created"); + int createdBy = rs.getInt("CreatedBy"); + // _ts + String description = rs.getString("Description"); + String type = rs.getString("Type"); + String title = rs.getString("Title"); + boolean searchable = rs.getBoolean("Searchable"); + String lockStateString = rs.getString("LockState"); + LockState lockState = null != lockStateString ? Enums.getIfPresent(LockState.class, lockStateString).or(LockState.Unlocked) : LockState.Unlocked; + + LocalDate expirationDate = rs.getObject("ExpirationDate", LocalDate.class); + + // Could be running upgrade code before these recent columns have been added to the table. Use a find map + // to determine if they are present. Issue 51692. These checks could be removed after creation of these + // columns is incorporated into the bootstrap scripts. + Map findMap = ResultSetUtil.getFindMap(rs.getMetaData()); + Long fileRootSize = findMap.containsKey("FileRootSize") ? (Long)rs.getObject("FileRootSize") : null; // getObject() and cast because getLong() returns 0 for null + LocalDateTime fileRootLastCrawled = findMap.containsKey("FileRootLastCrawled") ? rs.getObject("FileRootLastCrawled", LocalDateTime.class) : null; + + Container dirParent = null; + if (null != parentId) + dirParent = getForId(parentId); + + d = new Container(dirParent, name, id, rowId, sortOrder, created, createdBy, searchable); + d.setDescription(description); + d.setType(type); + d.setTitle(title); + d.setLockState(lockState); + d.setExpirationDate(expirationDate); + d.setFileRootSize(fileRootSize); + d.setFileRootLastCrawled(fileRootLastCrawled); + return d; + } + + @Override + public ArrayList handleArrayList(ResultSet rs) throws SQLException + { + ArrayList list = new ArrayList<>(); + while (rs.next()) + { + list.add(handle(rs)); + } + return list; + } + } + + public static Container createFakeContainer(@Nullable String name, @Nullable Container parent) + { + return new Container(parent, name, GUID.makeGUID(), 1, 0, new Date(), 0, false); + } +} diff --git a/api/src/org/labkey/api/exp/list/ListDefinition.java b/api/src/org/labkey/api/exp/list/ListDefinition.java index 20779eed733..95bc37c1a25 100644 --- a/api/src/org/labkey/api/exp/list/ListDefinition.java +++ b/api/src/org/labkey/api/exp/list/ListDefinition.java @@ -277,9 +277,11 @@ public static BodySetting getForValue(int value) int getListId(); void setPreferredListIds(Collection preferredListIds); // Attempts to use this list IDs when inserting Container getContainer(); - @Nullable Domain getDomain(); + @Nullable Domain getDomain(); @Nullable Domain getDomain(boolean forUpdate); + @NotNull Domain getDomainOrThrow(); + @NotNull Domain getDomainOrThrow(boolean forUpdate); String getName(); String getKeyName(); diff --git a/api/src/org/labkey/api/exp/list/ListService.java b/api/src/org/labkey/api/exp/list/ListService.java index 9d79a027e38..590c1b402de 100644 --- a/api/src/org/labkey/api/exp/list/ListService.java +++ b/api/src/org/labkey/api/exp/list/ListService.java @@ -1,65 +1,65 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.exp.list; - -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.exp.TemplateInfo; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.query.UserSchema; -import org.labkey.api.security.User; -import org.labkey.api.services.ServiceRegistry; -import org.labkey.api.view.ActionURL; -import org.springframework.validation.BindException; - -import java.io.InputStream; -import java.util.Map; - -public interface ListService -{ - static ListService get() - { - return ServiceRegistry.get().getService(ListService.class); - } - - static void setInstance(ListService ls) - { - ServiceRegistry.get().registerService(ListService.class, ls); - } - - Map getLists(Container container); - Map getLists(Container container, @Nullable User user); - Map getLists(Container container, @Nullable User user, boolean checkVisibility); - Map getLists(Container container, @Nullable User user, boolean checkVisibility, boolean includePicklists, boolean includeProjectAndShared); - boolean hasLists(Container container); - boolean hasLists(Container container, boolean includeProjectAndShared); - ListDefinition createList(Container container, String name, ListDefinition.KeyType keyType); - ListDefinition createList(Container container, String name, ListDefinition.KeyType keyType, @Nullable TemplateInfo templateInfo, @Nullable ListDefinition.Category category); - @Nullable ListDefinition getList(Container container, int listId); - @Nullable ListDefinition getList(Container container, String name); - @Nullable ListDefinition getList(Container container, String name, boolean includeProjectAndShared); - ListDefinition getList(Domain domain); - ActionURL getManageListsURL(Container container); - UserSchema getUserSchema(User user, Container container); - - /** Picklists can specify different container filtering configurations depending on the container context */ - @Nullable ContainerFilter getPicklistContainerFilter(Container container, User user, @NotNull ListDefinition list); - - void importListArchive(InputStream is, BindException errors, Container c, User user) throws Exception; -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.exp.list; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.exp.TemplateInfo; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; +import org.labkey.api.services.ServiceRegistry; +import org.labkey.api.view.ActionURL; +import org.springframework.validation.BindException; + +import java.io.InputStream; +import java.util.Map; + +public interface ListService +{ + static ListService get() + { + return ServiceRegistry.get().getService(ListService.class); + } + + static void setInstance(ListService ls) + { + ServiceRegistry.get().registerService(ListService.class, ls); + } + + Map getLists(Container container); + Map getLists(Container container, @Nullable User user); + Map getLists(Container container, @Nullable User user, boolean checkVisibility); + Map getLists(Container container, @Nullable User user, boolean checkVisibility, boolean includePicklists, boolean includeProjectAndShared); + boolean hasLists(Container container); + boolean hasLists(Container container, boolean includeProjectAndShared); + ListDefinition createList(Container container, String name, ListDefinition.KeyType keyType); + ListDefinition createList(Container container, String name, ListDefinition.KeyType keyType, @Nullable TemplateInfo templateInfo, @Nullable ListDefinition.Category category); + @Nullable ListDefinition getList(Container container, int listId); + @Nullable ListDefinition getList(Container container, String name); + @Nullable ListDefinition getList(Container container, String name, boolean includeProjectAndShared); + ListDefinition getList(Domain domain); + ActionURL getManageListsURL(Container container); + UserSchema getUserSchema(User user, Container container); + + /** Picklists can specify different container filtering configurations depending on the container context */ + @Nullable ContainerFilter getPicklistContainerFilter(Container container, User user, @NotNull ListDefinition list); + + void importListArchive(InputStream is, BindException errors, Container c, User user) throws Exception; +} diff --git a/api/src/org/labkey/api/query/QueryService.java b/api/src/org/labkey/api/query/QueryService.java index 1039003aaaa..922c5af62b9 100644 --- a/api/src/org/labkey/api/query/QueryService.java +++ b/api/src/org/labkey/api/query/QueryService.java @@ -462,7 +462,7 @@ public String getDefaultCommentSummary() List getQueryUpdateAuditRecords(User user, Container container, long transactionAuditId, @Nullable ContainerFilter containerFilter); AuditHandler getDefaultAuditHandler(); - int moveAuditEvents(Container targetContainer, List rowPks, String schemaName, String queryName); + int moveAuditEvents(Container targetContainer, List rowPks, String schemaName, String queryName); /** * Returns a URL for the audit history for the table. diff --git a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java index cb67b95ded2..93622d75b78 100644 --- a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java @@ -9635,8 +9635,8 @@ public Map moveDataClassObjects(Collection d throw errors; // move audit events associated with the sources that are moving - int auditEventCount = QueryService.get().moveAuditEvents(targetContainer, dataIds, "exp.data", dataClassTable.getName()); - updateCounts.compute("sourceAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount ); + int auditEventCount = QueryService.get().moveAuditEvents(targetContainer, List.of(dataIds), "exp.data", dataClassTable.getName()); + updateCounts.compute("sourceAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount); // create summary audit entries for the source container only. The message is pretty generic, so having it // in both source and target doesn't help much. diff --git a/list/src/org/labkey/list/model/ListAuditProvider.java b/list/src/org/labkey/list/model/ListAuditProvider.java index 3f25042cfd8..028f99770f9 100644 --- a/list/src/org/labkey/list/model/ListAuditProvider.java +++ b/list/src/org/labkey/list/model/ListAuditProvider.java @@ -44,10 +44,6 @@ import java.util.Map; import java.util.Set; -/** - * User: klum - * Date: 7/21/13 - */ public class ListAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider { public static final String COLUMN_NAME_LIST_ID = "ListId"; @@ -143,6 +139,11 @@ public Class getEventClass() return (Class)ListAuditEvent.class; } + public int moveEvents(Container targetContainer, List listRowEntityIds) + { + return moveEvents(targetContainer, COLUMN_NAME_LIST_ITEM_ENTITY_ID, listRowEntityIds); + } + public static class ListAuditEvent extends DetailedAuditTypeEvent { private int _listId; diff --git a/list/src/org/labkey/list/model/ListDefinitionImpl.java b/list/src/org/labkey/list/model/ListDefinitionImpl.java index 07c3b39a477..d341b032912 100644 --- a/list/src/org/labkey/list/model/ListDefinitionImpl.java +++ b/list/src/org/labkey/list/model/ListDefinitionImpl.java @@ -153,6 +153,22 @@ public Domain getDomain(boolean forUpdate) } return _domain; } + + @Override + public @NotNull Domain getDomainOrThrow() + { + return getDomainOrThrow(false); + } + + @Override + public @NotNull Domain getDomainOrThrow(boolean forUpdate) + { + var domain = getDomain(forUpdate); + if (domain == null) + throw new IllegalArgumentException("Could not find domain for list \"" + getName() + "\"."); + return domain; + } + @Override public String getName() { @@ -510,7 +526,7 @@ private ListItem getListItem(SimpleFilter filter, User user, Container c) itm.setKey(row.get(getKeyName())); ListItemImpl impl = new ListItemImpl(this, itm); - for (DomainProperty prop : getDomain().getProperties()) + for (DomainProperty prop : getDomainOrThrow().getProperties()) { impl.setProperty(prop, row.get(prop.getName())); } @@ -555,12 +571,12 @@ public void delete(User user, @Nullable String auditUserComment) throws DomainNo { // remove related attachments, discussions, and indices ListManager.get().deleteIndexedList(this); - if (qus instanceof ListQueryUpdateService) - ((ListQueryUpdateService)qus).deleteRelatedListData(user, getContainer()); + if (qus instanceof ListQueryUpdateService listQus) + listQus.deleteRelatedListData(user, getContainer()); // then delete the list itself ListManager.get().deleteListDef(getContainer(), getListId()); - Domain domain = getDomain(); + Domain domain = getDomainOrThrow(); domain.delete(user, auditUserComment); ListManager.get().addAuditEvent(this, user, String.format("The list %s was deleted", _def.getName())); @@ -669,7 +685,6 @@ public void setLastIndexed(Date modified) edit().setLastIndexed(modified); } - /** NOTE consider using ListQuerySchema.getTable(), unless you have a good reason */ @Override @Nullable public TableInfo getTable(User user) @@ -677,7 +692,6 @@ public TableInfo getTable(User user) return getTable(user, getContainer()); } - /** NOTE consider using ListQuerySchema.getTable(), unless you have a good reason */ @Override @Nullable public TableInfo getTable(User user, Container c) diff --git a/list/src/org/labkey/list/model/ListManager.java b/list/src/org/labkey/list/model/ListManager.java index ea2b6a046cc..497721ba2b4 100644 --- a/list/src/org/labkey/list/model/ListManager.java +++ b/list/src/org/labkey/list/model/ListManager.java @@ -100,7 +100,6 @@ public class ListManager implements SearchService.DocumentProvider public static final String LIST_AUDIT_EVENT = "ListAuditEvent"; public static final String LISTID_FIELD_NAME = "listId"; - private final Cache> _listDefCache = DatabaseCache.get(CoreSchema.getInstance().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "List definitions", new ListDefCacheLoader()) ; private class ListDefCacheLoader implements CacheLoader> @@ -1141,9 +1140,6 @@ void addAuditEvent(ListDefinitionImpl list, User user, String comment) } } - /** - * Modeled after ListItemImpl.addAuditEvent - */ void addAuditEvent(ListDefinitionImpl list, User user, Container c, String comment, String entityId, @Nullable String oldRecord, @Nullable String newRecord) { ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(c, comment, list); diff --git a/list/src/org/labkey/list/model/ListQueryUpdateService.java b/list/src/org/labkey/list/model/ListQueryUpdateService.java index c6a0609682b..83b91f91a88 100644 --- a/list/src/org/labkey/list/model/ListQueryUpdateService.java +++ b/list/src/org/labkey/list/model/ListQueryUpdateService.java @@ -1,612 +1,841 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.list.model; - -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.announcements.DiscussionService; -import org.labkey.api.attachments.AttachmentFile; -import org.labkey.api.attachments.AttachmentParent; -import org.labkey.api.attachments.AttachmentParentFactory; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.LookupResolutionType; -import org.labkey.api.data.Selector.ForEachBatchBlock; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.dataiterator.DataIteratorContext; -import org.labkey.api.exp.ObjectProperty; -import org.labkey.api.exp.list.ListDefinition; -import org.labkey.api.exp.list.ListImportProgress; -import org.labkey.api.exp.list.ListItem; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.IPropertyValidator; -import org.labkey.api.exp.property.ValidatorContext; -import org.labkey.api.lists.permissions.ManagePicklistsPermission; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.DefaultQueryUpdateService; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.InvalidKeyException; -import org.labkey.api.query.PropertyValidationError; -import org.labkey.api.query.QueryUpdateServiceException; -import org.labkey.api.query.ValidationError; -import org.labkey.api.query.ValidationException; -import org.labkey.api.reader.DataLoader; -import org.labkey.api.security.ElevatedUser; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.security.roles.EditorRole; -import org.labkey.api.util.Pair; -import org.labkey.api.util.UnexpectedException; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.writer.VirtualFile; -import org.labkey.list.view.ListItemAttachmentParent; - -import java.io.IOException; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.labkey.api.util.IntegerUtils.isIntegral; - -/** - * Implementation of QueryUpdateService for Lists - */ -public class ListQueryUpdateService extends DefaultQueryUpdateService -{ - private final ListDefinitionImpl _list; - private static final String ID = "entityId"; - - public ListQueryUpdateService(ListTable queryTable, TableInfo dbTable, @NotNull ListDefinition list) - { - super(queryTable, dbTable, createMVMapping(queryTable.getList().getDomain())); - _list = (ListDefinitionImpl) list; - } - - @Override - public void configureDataIteratorContext(DataIteratorContext context) - { - if (context.getInsertOption().batch) - { - context.setMaxRowErrors(100); - context.setFailFast(false); - } - - context.putConfigParameter(ConfigParameters.TrimStringRight, Boolean.TRUE); - } - - @Override - protected @Nullable AttachmentParentFactory getAttachmentParentFactory() - { - return new ListItemAttachmentParentFactory(); - } - - @Override - protected Map getRow(User user, Container container, Map listRow) throws InvalidKeyException - { - Map ret = null; - - if (null != listRow) - { - SimpleFilter keyFilter = getKeyFilter(listRow); - - if (null != keyFilter) - { - TableInfo queryTable = getQueryTable(); - Map raw = new TableSelector(queryTable, keyFilter, null).getMap(); - - if (null != raw && !raw.isEmpty()) - { - ret = new CaseInsensitiveHashMap<>(); - - // EntityId - ret.put("EntityId", raw.get("entityid")); - - for (DomainProperty prop : _list.getDomain().getProperties()) - { - String propName = prop.getName(); - ColumnInfo column = queryTable.getColumn(propName); - Object value = column.getValue(raw); - if (value != null) - ret.put(propName, value); - } - } - } - } - - return ret; - } - - - @Override - public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, - @Nullable Map configParameters, Map extraScriptContext) - { - for (Map row : rows) - { - aliasColumns(getColumnMapping(), row); - } - - DataIteratorContext context = getDataIteratorContext(errors, InsertOption.INSERT, configParameters); - List> result = this._insertRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); - - if (null != result) - { - ListManager mgr = ListManager.get(); - - for (Map row : result) - { - if (null != row.get(ID)) - { - // Audit each row - String entityId = (String) row.get(ID); - String newRecord = mgr.formatAuditItem(_list, user, row); - - mgr.addAuditEvent(_list, user, container, "A new list record was inserted", entityId, null, newRecord); - } - } - - if (!result.isEmpty() && !errors.hasErrors()) - mgr.indexList(_list); - } - - return result; - } - - private User getListUser(User user, Container container) - { - if (_list.isPicklist() && container.hasPermission(user, ManagePicklistsPermission.class)) - { - // if the list is a picklist and you have permission to manage picklists, that equates - // to having editor permission. - return ElevatedUser.ensureContextualRoles(container, user, Pair.of(DeletePermission.class, EditorRole.class)); - } - return user; - } - - @Override - protected @Nullable List> _insertRowsUsingDIB(User user, Container container, List> rows, DataIteratorContext context, @Nullable Map extraScriptContext) - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to insert data into this table."); - - return super._insertRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); - } - - @Override - protected @Nullable List> _updateRowsUsingDIB(User user, Container container, List> rows, - DataIteratorContext context, @Nullable Map extraScriptContext) - { - if (!hasPermission(user, UpdatePermission.class)) - throw new UnauthorizedException("You do not have permission to update data in this table."); - - return super._updateRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); - } - - public int insertUsingDataIterator(DataLoader loader, User user, Container container, BatchValidationException errors, @Nullable VirtualFile attachmentDir, - @Nullable ListImportProgress progress, boolean supportAutoIncrementKey, InsertOption insertOption, LookupResolutionType lookupResolutionType) - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to insert data into this table."); - - User updatedUser = getListUser(user, container); - DataIteratorContext context = new DataIteratorContext(errors); - context.setFailFast(false); - context.setInsertOption(insertOption); // this method is used by ListImporter and BackgroundListImporter - context.setSupportAutoIncrementKey(supportAutoIncrementKey); - context.setLookupResolutionType(lookupResolutionType); - setAttachmentDirectory(attachmentDir); - TableInfo ti = _list.getTable(updatedUser); - - if (null != ti) - { - try (DbScope.Transaction transaction = ti.getSchema().getScope().ensureTransaction()) - { - int imported = _importRowsUsingDIB(updatedUser, container, loader, null, context, new HashMap<>()); - - if (!errors.hasErrors()) - { - //Make entry to audit log if anything was inserted - if (imported > 0) - ListManager.get().addAuditEvent(_list, updatedUser, "Bulk " + (insertOption.updateOnly ? "updated " : (insertOption.mergeRows ? "imported " : "inserted ")) + imported + " rows to list."); - - transaction.addCommitTask(() -> ListManager.get().indexList(_list), DbScope.CommitTaskOption.POSTCOMMIT); - transaction.commit(); - - return imported; - } - - return 0; - } - } - - return 0; - } - - - @Override - public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, - @Nullable Map configParameters, Map extraScriptContext) - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to update data in this table."); - - return _importRowsUsingDIB(getListUser(user, container), container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); - } - - - @Override - public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, - Map configParameters, Map extraScriptContext) - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to insert data into this table."); - - DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); - int count = _importRowsUsingDIB(getListUser(user, container), container, rows, null, context, extraScriptContext); - if (count > 0 && !errors.hasErrors()) - ListManager.get().indexList(_list); - return count; - } - - @Override - public List> updateRows(User user, Container container, List> rows, List> oldKeys, - BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) - throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to update data into this table."); - - List> result = super.updateRows(getListUser(user, container), container, rows, oldKeys, errors, configParameters, extraScriptContext); - if (!result.isEmpty()) - ListManager.get().indexList(_list); - return result; - } - - @Override - protected Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, @Nullable Map configParameters) - throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException - { - // TODO: Check for equivalency so that attachments can be deleted etc. - - Map dps = new HashMap<>(); - for (DomainProperty dp : _list.getDomain().getProperties()) - { - dps.put(dp.getPropertyURI(), dp); - } - - ValidatorContext validatorCache = new ValidatorContext(container, user); - - ListItm itm = new ListItm(); - itm.setEntityId((String) oldRow.get(ID)); - itm.setListId(_list.getListId()); - itm.setKey(oldRow.get(_list.getKeyName())); - - ListItem item = new ListItemImpl(_list, itm); - - if (item.getProperties() != null) - { - List errors = new ArrayList<>(); - for (Map.Entry entry : dps.entrySet()) - { - Object value = row.get(entry.getValue().getName()); - validateProperty(entry.getValue(), value, row, errors, validatorCache); - } - - if (!errors.isEmpty()) - throw new ValidationException(errors); - } - - // MVIndicators - Map rowCopy = new CaseInsensitiveHashMap<>(); - ArrayList modifiedAttachmentColumns = new ArrayList<>(); - ArrayList attachmentFiles = new ArrayList<>(); - - TableInfo qt = getQueryTable(); - for (Map.Entry r : row.entrySet()) - { - ColumnInfo column = qt.getColumn(FieldKey.fromParts(r.getKey())); - rowCopy.put(r.getKey(), r.getValue()); - - // 22747: Attachment columns - if (null != column) - { - DomainProperty dp = _list.getDomain().getPropertyByURI(column.getPropertyURI()); - if (null != dp && isAttachmentProperty(dp)) - { - modifiedAttachmentColumns.add(column); - - // setup any new attachments - if (r.getValue() instanceof AttachmentFile file) - { - if (null != file.getFilename()) - attachmentFiles.add(file); - } - else if (r.getValue() != null && !StringUtils.isEmpty(String.valueOf(r.getValue()))) - { - throw new ValidationException("Can't upload '" + r.getValue() + "' to field " + r.getKey() + " with type Attachment."); - } - } - } - } - - // Attempt to include key from oldRow if not found in row (As stated in the QUS Interface) - Object newRowKey = getField(rowCopy, _list.getKeyName()); - Object oldRowKey = getField(oldRow, _list.getKeyName()); - - if (null == newRowKey && null != oldRowKey) - rowCopy.put(_list.getKeyName(), oldRowKey); - - Map result = super.updateRow(getListUser(user, container), container, rowCopy, oldRow, true, false); - - if (null != result) - { - result = getRow(user, container, result); - - if (null != result && null != result.get(ID)) - { - ListManager mgr = ListManager.get(); - String entityId = (String) result.get(ID); - - try - { - // Remove prior attachment -- only includes columns which are modified in this update - for (ColumnInfo col : modifiedAttachmentColumns) - { - Object value = oldRow.get(col.getName()); - if (null != value) - { - AttachmentService.get().deleteAttachment(new ListItemAttachmentParent(entityId, _list.getContainer()), value.toString(), user); - } - } - - // Update attachments - if (!attachmentFiles.isEmpty()) - AttachmentService.get().addAttachments(new ListItemAttachmentParent(entityId, _list.getContainer()), attachmentFiles, user); - } - catch (AttachmentService.DuplicateFilenameException | AttachmentService.FileTooLargeException e) - { - // issues 21503, 28633: turn these into a validation exception to get a nicer error - throw new ValidationException(e.getMessage()); - } - catch (IOException e) - { - throw UnexpectedException.wrap(e); - } - finally - { - for (AttachmentFile attachmentFile : attachmentFiles) - { - try { attachmentFile.closeInputStream(); } catch (IOException ignored) {} - } - } - - String oldRecord = mgr.formatAuditItem(_list, user, oldRow); - String newRecord = mgr.formatAuditItem(_list, user, result); - - // Audit - mgr.addAuditEvent(_list, user, container, "An existing list record was modified", entityId, oldRecord, newRecord); - } - } - - return result; - } - - // TODO: Consolidate with ColumnValidator and OntologyManager.validateProperty() - private boolean validateProperty(DomainProperty prop, Object value, Map newRow, List errors, ValidatorContext validatorCache) - { - //check for isRequired - if (prop.isRequired()) - { - // for mv indicator columns either an indicator or a field value is sufficient - boolean hasMvIndicator = prop.isMvEnabled() && (value instanceof ObjectProperty && ((ObjectProperty)value).getMvIndicator() != null); - if (!hasMvIndicator && (null == value || (value instanceof ObjectProperty && ((ObjectProperty)value).value() == null))) - { - if (newRow.containsKey(prop.getName()) && newRow.get(prop.getName()) == null) - { - errors.add(new PropertyValidationError("The field '" + prop.getName() + "' is required.", prop.getName())); - return false; - } - } - } - - if (null != value) - { - for (IPropertyValidator validator : prop.getValidators()) - { - if (!validator.validate(prop.getPropertyDescriptor(), value, errors, validatorCache)) - return false; - } - } - - return true; - } - - @Override - protected Map deleteRow(User user, Container container, Map oldRowMap) throws InvalidKeyException, QueryUpdateServiceException, SQLException - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to delete data from this table."); - - Map result = super.deleteRow(getListUser(user, container), container, oldRowMap); - - if (null != result) - { - String entityId = (String) result.get(ID); - - if (null != entityId) - { - ListManager mgr = ListManager.get(); - String deletedRecord = mgr.formatAuditItem(_list, user, result); - - // Audit - mgr.addAuditEvent(_list, user, container, "An existing list record was deleted", entityId, deletedRecord, null); - - // Remove discussions - if (DiscussionService.get() != null) - DiscussionService.get().deleteDiscussions(container, user, entityId); - - // Remove attachments - if (hasAttachmentProperties()) - AttachmentService.get().deleteAttachments(new ListItemAttachmentParent(entityId, container)); - - // Clean up Search indexer - if (!result.isEmpty()) - mgr.deleteItemIndex(_list, entityId); - } - } - - return result; - } - - - // Deletes attachments & discussions, and removes list documents from full-text search index. - public void deleteRelatedListData(final User user, final Container container) - { - // Unindex all item docs and the entire list doc - ListManager.get().deleteIndexedList(_list); - - // Delete attachments and discussions associated with a list in batches of 1,000 - new TableSelector(getDbTable(), Collections.singleton("entityId")).forEachBatch(String.class, 1000, new ForEachBatchBlock<>() - { - @Override - public boolean accept(String entityId) - { - return null != entityId; - } - - @Override - public void exec(List entityIds) - { - // delete the related list data for this block - deleteRelatedListData(user, container, entityIds); - } - }); - } - - // delete the related list data for this block of entityIds - private void deleteRelatedListData(User user, Container container, List entityIds) - { - // Build up set of entityIds and AttachmentParents - List attachmentParents = new ArrayList<>(); - - // Delete Discussions - if (_list.getDiscussionSetting() != ListDefinition.DiscussionSetting.None && DiscussionService.get() != null) - DiscussionService.get().deleteDiscussions(container, user, entityIds); - - // Delete Attachments - if (hasAttachmentProperties()) - { - for (String entityId : entityIds) - { - attachmentParents.add(new ListItemAttachmentParent(entityId, container)); - } - AttachmentService.get().deleteAttachments(attachmentParents); - } - } - - @Override - protected int truncateRows(User user, Container container) throws QueryUpdateServiceException, SQLException - { - int result; - try (DbScope.Transaction transaction = getDbTable().getSchema().getScope().ensureTransaction()) - { - deleteRelatedListData(user, container); - result = super.truncateRows(getListUser(user, container), container); - transaction.addCommitTask(() -> ListManager.get().addAuditEvent(_list, user, "Deleted " + result + " rows from list."), DbScope.CommitTaskOption.POSTCOMMIT); - transaction.commit(); - } - - return result; - } - - - @Nullable - public SimpleFilter getKeyFilter(Map map) throws InvalidKeyException - { - String keyName = _list.getKeyName(); - ListDefinition.KeyType type = _list.getKeyType(); - - Object key = getField(map, _list.getKeyName()); - - if (null == key) - { - // Auto-increment lists might not provide a key so allow them to pass through - if (type.equals(ListDefinition.KeyType.AutoIncrementInteger)) - return null; - throw new InvalidKeyException("No " + keyName + " provided for list \"" + _list.getName() + "\""); - } - - // Check the type of the list to ensure proper casting of the key type - if (type.equals(ListDefinition.KeyType.Integer) || type.equals(ListDefinition.KeyType.AutoIncrementInteger)) - { - if (isIntegral(key)) - return new SimpleFilter(FieldKey.fromParts(keyName), key); - return new SimpleFilter(FieldKey.fromParts(keyName), Integer.valueOf(key.toString())); - } - - return new SimpleFilter(FieldKey.fromParts(keyName), key.toString()); - } - - @Nullable - private Object getField(Map map, String key) - { - /* TODO: this is very strange, we have a TableInfo we should be using its ColumnInfo objects to figure out aliases, we don't need to guess */ - Object value = map.get(key); - - if (null == value) - value = map.get(key + "_"); - - if (null == value) - value = map.get(getDbTable().getSqlDialect().legalNameFromName(key)); - - return value; - } - - /** - * Delegate class to generate an AttachmentParent - */ - public static class ListItemAttachmentParentFactory implements AttachmentParentFactory - { - @Override - public AttachmentParent generateAttachmentParent(String entityId, Container c) - { - return new ListItemAttachmentParent(entityId, c); - } - } - - /** - * Get Domain from list definition, unless null then get from super - */ - @Override - protected Domain getDomain() - { - return _list != null? - _list.getDomain() : - super.getDomain(); - } -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.list.model; + +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.announcements.DiscussionService; +import org.labkey.api.attachments.AttachmentFile; +import org.labkey.api.attachments.AttachmentParent; +import org.labkey.api.attachments.AttachmentParentFactory; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.LookupResolutionType; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.Selector.ForEachBatchBlock; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; +import org.labkey.api.exp.ObjectProperty; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.list.ListDefinition; +import org.labkey.api.exp.list.ListImportProgress; +import org.labkey.api.exp.list.ListItem; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.IPropertyValidator; +import org.labkey.api.exp.property.ValidatorContext; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.lists.permissions.ManagePicklistsPermission; +import org.labkey.api.query.AbstractQueryUpdateService; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.DefaultQueryUpdateService; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.InvalidKeyException; +import org.labkey.api.query.PropertyValidationError; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateServiceException; +import org.labkey.api.query.ValidationError; +import org.labkey.api.query.ValidationException; +import org.labkey.api.reader.DataLoader; +import org.labkey.api.security.ElevatedUser; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.MoveEntitiesPermission; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.security.roles.EditorRole; +import org.labkey.api.usageMetrics.SimpleMetricsService; +import org.labkey.api.util.GUID; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.UnexpectedException; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.writer.VirtualFile; +import org.labkey.list.view.ListItemAttachmentParent; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.labkey.api.util.IntegerUtils.isIntegral; + +/** + * Implementation of QueryUpdateService for Lists + */ +public class ListQueryUpdateService extends DefaultQueryUpdateService +{ + private final ListDefinitionImpl _list; + private static final String ID = "entityId"; + + public ListQueryUpdateService(ListTable queryTable, TableInfo dbTable, @NotNull ListDefinition list) + { + super(queryTable, dbTable, createMVMapping(queryTable.getList().getDomain())); + _list = (ListDefinitionImpl) list; + } + + @Override + public void configureDataIteratorContext(DataIteratorContext context) + { + if (context.getInsertOption().batch) + { + context.setMaxRowErrors(100); + context.setFailFast(false); + } + + context.putConfigParameter(ConfigParameters.TrimStringRight, Boolean.TRUE); + } + + @Override + protected @Nullable AttachmentParentFactory getAttachmentParentFactory() + { + return new ListItemAttachmentParentFactory(); + } + + @Override + protected Map getRow(User user, Container container, Map listRow) throws InvalidKeyException + { + Map ret = null; + + if (null != listRow) + { + SimpleFilter keyFilter = getKeyFilter(listRow); + + if (null != keyFilter) + { + TableInfo queryTable = getQueryTable(); + Map raw = new TableSelector(queryTable, keyFilter, null).getMap(); + + if (null != raw && !raw.isEmpty()) + { + ret = new CaseInsensitiveHashMap<>(); + + // EntityId + ret.put("EntityId", raw.get("entityid")); + + for (DomainProperty prop : _list.getDomainOrThrow().getProperties()) + { + String propName = prop.getName(); + ColumnInfo column = queryTable.getColumn(propName); + Object value = column.getValue(raw); + if (value != null) + ret.put(propName, value); + } + } + } + } + + return ret; + } + + + @Override + public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, + @Nullable Map configParameters, Map extraScriptContext) + { + for (Map row : rows) + { + aliasColumns(getColumnMapping(), row); + } + + DataIteratorContext context = getDataIteratorContext(errors, InsertOption.INSERT, configParameters); + List> result = this._insertRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); + + if (null != result) + { + ListManager mgr = ListManager.get(); + + for (Map row : result) + { + if (null != row.get(ID)) + { + // Audit each row + String entityId = (String) row.get(ID); + String newRecord = mgr.formatAuditItem(_list, user, row); + + mgr.addAuditEvent(_list, user, container, "A new list record was inserted", entityId, null, newRecord); + } + } + + if (!result.isEmpty() && !errors.hasErrors()) + mgr.indexList(_list); + } + + return result; + } + + private User getListUser(User user, Container container) + { + if (_list.isPicklist() && container.hasPermission(user, ManagePicklistsPermission.class)) + { + // if the list is a picklist and you have permission to manage picklists, that equates + // to having editor permission. + return ElevatedUser.ensureContextualRoles(container, user, Pair.of(DeletePermission.class, EditorRole.class)); + } + return user; + } + + @Override + protected @Nullable List> _insertRowsUsingDIB(User user, Container container, List> rows, DataIteratorContext context, @Nullable Map extraScriptContext) + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to insert data into this table."); + + return super._insertRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); + } + + @Override + protected @Nullable List> _updateRowsUsingDIB(User user, Container container, List> rows, + DataIteratorContext context, @Nullable Map extraScriptContext) + { + if (!hasPermission(user, UpdatePermission.class)) + throw new UnauthorizedException("You do not have permission to update data in this table."); + + return super._updateRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); + } + + public int insertUsingDataIterator(DataLoader loader, User user, Container container, BatchValidationException errors, @Nullable VirtualFile attachmentDir, + @Nullable ListImportProgress progress, boolean supportAutoIncrementKey, InsertOption insertOption, LookupResolutionType lookupResolutionType) + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to insert data into this table."); + + User updatedUser = getListUser(user, container); + DataIteratorContext context = new DataIteratorContext(errors); + context.setFailFast(false); + context.setInsertOption(insertOption); // this method is used by ListImporter and BackgroundListImporter + context.setSupportAutoIncrementKey(supportAutoIncrementKey); + context.setLookupResolutionType(lookupResolutionType); + setAttachmentDirectory(attachmentDir); + TableInfo ti = _list.getTable(updatedUser); + + if (null != ti) + { + try (DbScope.Transaction transaction = ti.getSchema().getScope().ensureTransaction()) + { + int imported = _importRowsUsingDIB(updatedUser, container, loader, null, context, new HashMap<>()); + + if (!errors.hasErrors()) + { + //Make entry to audit log if anything was inserted + if (imported > 0) + ListManager.get().addAuditEvent(_list, updatedUser, "Bulk " + (insertOption.updateOnly ? "updated " : (insertOption.mergeRows ? "imported " : "inserted ")) + imported + " rows to list."); + + transaction.addCommitTask(() -> ListManager.get().indexList(_list), DbScope.CommitTaskOption.POSTCOMMIT); + transaction.commit(); + + return imported; + } + + return 0; + } + } + + return 0; + } + + + @Override + public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, + @Nullable Map configParameters, Map extraScriptContext) + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to update data in this table."); + + return _importRowsUsingDIB(getListUser(user, container), container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); + } + + + @Override + public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, + Map configParameters, Map extraScriptContext) + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to insert data into this table."); + + DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); + int count = _importRowsUsingDIB(getListUser(user, container), container, rows, null, context, extraScriptContext); + if (count > 0 && !errors.hasErrors()) + ListManager.get().indexList(_list); + return count; + } + + @Override + public List> updateRows(User user, Container container, List> rows, List> oldKeys, + BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) + throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to update data into this table."); + + List> result = super.updateRows(getListUser(user, container), container, rows, oldKeys, errors, configParameters, extraScriptContext); + if (!result.isEmpty()) + ListManager.get().indexList(_list); + return result; + } + + @Override + protected Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, @Nullable Map configParameters) + throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException + { + // TODO: Check for equivalency so that attachments can be deleted etc. + + Map dps = new HashMap<>(); + for (DomainProperty dp : _list.getDomainOrThrow().getProperties()) + { + dps.put(dp.getPropertyURI(), dp); + } + + ValidatorContext validatorCache = new ValidatorContext(container, user); + + ListItm itm = new ListItm(); + itm.setEntityId((String) oldRow.get(ID)); + itm.setListId(_list.getListId()); + itm.setKey(oldRow.get(_list.getKeyName())); + + ListItem item = new ListItemImpl(_list, itm); + + if (item.getProperties() != null) + { + List errors = new ArrayList<>(); + for (Map.Entry entry : dps.entrySet()) + { + Object value = row.get(entry.getValue().getName()); + validateProperty(entry.getValue(), value, row, errors, validatorCache); + } + + if (!errors.isEmpty()) + throw new ValidationException(errors); + } + + // MVIndicators + Map rowCopy = new CaseInsensitiveHashMap<>(); + ArrayList modifiedAttachmentColumns = new ArrayList<>(); + ArrayList attachmentFiles = new ArrayList<>(); + + TableInfo qt = getQueryTable(); + for (Map.Entry r : row.entrySet()) + { + ColumnInfo column = qt.getColumn(FieldKey.fromParts(r.getKey())); + rowCopy.put(r.getKey(), r.getValue()); + + // 22747: Attachment columns + if (null != column) + { + DomainProperty dp = _list.getDomainOrThrow().getPropertyByURI(column.getPropertyURI()); + if (null != dp && isAttachmentProperty(dp)) + { + modifiedAttachmentColumns.add(column); + + // setup any new attachments + if (r.getValue() instanceof AttachmentFile file) + { + if (null != file.getFilename()) + attachmentFiles.add(file); + } + else if (r.getValue() != null && !StringUtils.isEmpty(String.valueOf(r.getValue()))) + { + throw new ValidationException("Can't upload '" + r.getValue() + "' to field " + r.getKey() + " with type Attachment."); + } + } + } + } + + // Attempt to include key from oldRow if not found in row (As stated in the QUS Interface) + Object newRowKey = getField(rowCopy, _list.getKeyName()); + Object oldRowKey = getField(oldRow, _list.getKeyName()); + + if (null == newRowKey && null != oldRowKey) + rowCopy.put(_list.getKeyName(), oldRowKey); + + Map result = super.updateRow(getListUser(user, container), container, rowCopy, oldRow, true, false); + + if (null != result) + { + result = getRow(user, container, result); + + if (null != result && null != result.get(ID)) + { + ListManager mgr = ListManager.get(); + String entityId = (String) result.get(ID); + + try + { + // Remove prior attachment -- only includes columns which are modified in this update + for (ColumnInfo col : modifiedAttachmentColumns) + { + Object value = oldRow.get(col.getName()); + if (null != value) + { + AttachmentService.get().deleteAttachment(new ListItemAttachmentParent(entityId, _list.getContainer()), value.toString(), user); + } + } + + // Update attachments + if (!attachmentFiles.isEmpty()) + AttachmentService.get().addAttachments(new ListItemAttachmentParent(entityId, _list.getContainer()), attachmentFiles, user); + } + catch (AttachmentService.DuplicateFilenameException | AttachmentService.FileTooLargeException e) + { + // issues 21503, 28633: turn these into a validation exception to get a nicer error + throw new ValidationException(e.getMessage()); + } + catch (IOException e) + { + throw UnexpectedException.wrap(e); + } + finally + { + for (AttachmentFile attachmentFile : attachmentFiles) + { + try { attachmentFile.closeInputStream(); } catch (IOException ignored) {} + } + } + + String oldRecord = mgr.formatAuditItem(_list, user, oldRow); + String newRecord = mgr.formatAuditItem(_list, user, result); + + // Audit + mgr.addAuditEvent(_list, user, container, "An existing list record was modified", entityId, oldRecord, newRecord); + } + } + + return result; + } + + // TODO: Consolidate with ColumnValidator and OntologyManager.validateProperty() + private boolean validateProperty(DomainProperty prop, Object value, Map newRow, List errors, ValidatorContext validatorCache) + { + //check for isRequired + if (prop.isRequired()) + { + // for mv indicator columns either an indicator or a field value is sufficient + boolean hasMvIndicator = prop.isMvEnabled() && (value instanceof ObjectProperty && ((ObjectProperty)value).getMvIndicator() != null); + if (!hasMvIndicator && (null == value || (value instanceof ObjectProperty && ((ObjectProperty)value).value() == null))) + { + if (newRow.containsKey(prop.getName()) && newRow.get(prop.getName()) == null) + { + errors.add(new PropertyValidationError("The field '" + prop.getName() + "' is required.", prop.getName())); + return false; + } + } + } + + if (null != value) + { + for (IPropertyValidator validator : prop.getValidators()) + { + if (!validator.validate(prop.getPropertyDescriptor(), value, errors, validatorCache)) + return false; + } + } + + return true; + } + + private record ListRecord(Object key, String entityId) { } + + @Override + public Map moveRows( + User _user, + Container container, + Container targetContainer, + List> rows, + BatchValidationException errors, + @Nullable Map configParameters, + @Nullable Map extraScriptContext + ) throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException + { + Map updateCounts = new HashMap<>(); + updateCounts.put("listAuditEventsCreated", 0); + updateCounts.put("listAuditEventsMoved", 0); + updateCounts.put("listRecords", 0); + updateCounts.put("queryAuditEventsMoved", 0); + + Map> containerRows = getListRowsForMoveRows(targetContainer, rows, errors); + if (errors.hasErrors() || containerRows.isEmpty()) + return updateCounts; + + AuditBehaviorType auditBehavior = configParameters != null ? (AuditBehaviorType) configParameters.get(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior) : null; + String auditUserComment = configParameters != null ? (String) configParameters.get(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment) : null; + User user = getListUser(_user, container); + String listSchemaName = ListSchema.getInstance().getSchemaName(); + boolean hasAttachmentProperties = _list.getDomainOrThrow() + .getProperties() + .stream() + .anyMatch(prop -> PropertyType.ATTACHMENT.equals(prop.getPropertyType())); + + int listAuditEventsCreatedCount = 0; + int listAuditEventsMovedCount = 0; + int listRecordsCount = 0; + int queryAuditEventsMovedCount = 0; + ListAuditProvider listAuditProvider = new ListAuditProvider(); + + try (DbScope.Transaction tx = getDbTable().getSchema().getScope().ensureTransaction()) + { + if (auditBehavior != null && AuditBehaviorType.NONE != auditBehavior && tx.getAuditEvent() == null) + { + TransactionAuditProvider.TransactionAuditEvent auditEvent = createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); + auditEvent.updateCommentRowCount(containerRows.values().stream().mapToInt(List::size).sum()); + AbstractQueryUpdateService.addTransactionAuditEvent(tx, user, auditEvent); + } + + List listAuditEvents = new ArrayList<>(); + + for (GUID containerId : containerRows.keySet()) + { + Container sourceContainer = ContainerManager.getForId(containerId); + if (sourceContainer == null) + throw new InvalidKeyException("Container '" + containerId + "' does not exist."); + + if (!sourceContainer.hasPermission(user, MoveEntitiesPermission.class)) + throw new UnauthorizedException("You do not have permission to move list records out of '" + sourceContainer.getName() + "'."); + + TableInfo listTable = _list.getTable(user, sourceContainer); + if (listTable == null) + throw new QueryUpdateServiceException(String.format("Failed to retrieve table for list '%s' in folder %s.", _list.getName(), sourceContainer.getPath())); + + List records = containerRows.get(containerId); + List rowPks = records.stream().map(ListRecord::key).toList(); + + Map extraContext = Map.of("targetContainer", targetContainer, "keys", rowPks); + listTable.fireBatchTrigger(sourceContainer, user, TableInfo.TriggerType.MOVE, true, errors, extraContext); + if (errors.hasErrors()) + return updateCounts; + + listRecordsCount += ContainerManager.updateContainer(getDbTable(), _list.getKeyName(), rowPks, targetContainer, user, true); + if (errors.hasErrors()) + return updateCounts; + + if (hasAttachmentProperties) + { + moveAttachments(user, sourceContainer, targetContainer, records, errors); + if (errors.hasErrors()) + return updateCounts; + } + + queryAuditEventsMovedCount += QueryService.get().moveAuditEvents(targetContainer, rowPks, listSchemaName, _list.getName()); + listAuditEventsMovedCount += listAuditProvider.moveEvents(targetContainer, records.stream().map(ListRecord::entityId).toList()); + + // Create a summary audit event for the source container + { + String comment = String.format("Moved %s to %s", StringUtilsLabKey.pluralize(records.size(), "row"), targetContainer.getPath()); + ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(sourceContainer, comment, _list); + event.setUserComment(auditUserComment); + listAuditEvents.add(event); + } + + // Create a summary audit event for the target container + { + String comment = String.format("Moved %s from %s", StringUtilsLabKey.pluralize(records.size(), "row"), sourceContainer.getPath()); + ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(targetContainer, comment, _list); + event.setUserComment(auditUserComment); + listAuditEvents.add(event); + } + + if (AuditBehaviorType.DETAILED == listTable.getEffectiveAuditBehavior(auditBehavior)) + listAuditEventsCreatedCount += addDetailedMoveAuditEvents(user, sourceContainer, targetContainer, records); + + // TODO: Picklist support? + + listTable.fireBatchTrigger(sourceContainer, user, TableInfo.TriggerType.MOVE, false, errors, extraContext); + if (errors.hasErrors()) + return updateCounts; + } + + if (!listAuditEvents.isEmpty()) + { + AuditLogService.get().addEvents(user, listAuditEvents, true); + listAuditEventsCreatedCount += listAuditEvents.size(); + } + + tx.addCommitTask(() -> ListManager.get().indexList(_list), DbScope.CommitTaskOption.POSTCOMMIT); + + tx.commit(); + + SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "moveEntities", "list"); + } + + updateCounts.put("listAuditEventsCreated", listAuditEventsCreatedCount); + updateCounts.put("listAuditEventsMoved", listAuditEventsMovedCount); + updateCounts.put("listRecords", listRecordsCount); + updateCounts.put("queryAuditEventsMoved", queryAuditEventsMovedCount); + + return updateCounts; + } + + private Map> getListRowsForMoveRows(Container targetContainer, List> rows, BatchValidationException errors) + { + if (rows.isEmpty()) + return Collections.emptyMap(); + + String keyName = _list.getKeyName(); + List keys = new ArrayList<>(); + for (var row : rows) + { + Object key = getField(row, keyName); + if (key == null) + { + errors.addRowError(new ValidationException("Key field value required for moving list rows.")); + return Collections.emptyMap(); + } + + keys.add(getKeyFilterValue(key)); + } + + SimpleFilter filter = new SimpleFilter(); + FieldKey fieldKey = FieldKey.fromParts(keyName); + filter.addInClause(fieldKey, keys); + filter.addCondition(FieldKey.fromParts("Container"), targetContainer.getId(), CompareType.NEQ); + + Map> containerRows = new HashMap<>(); + try (var result = new TableSelector(getQueryTable(), PageFlowUtil.set(keyName, "Container", "EntityId"), filter, null).getResults()) + { + while (result.next()) + { + GUID containerId = new GUID(result.getString("Container")); + containerRows.computeIfAbsent(containerId, k -> new ArrayList<>()); + containerRows.get(containerId).add(new ListRecord(result.getObject(fieldKey), result.getString("EntityId"))); + } + } + catch (SQLException e) + { + throw new RuntimeSQLException(e); + } + + return containerRows; + } + + private void moveAttachments(User user, Container sourceContainer, Container targetContainer, List records, BatchValidationException errors) + { + // TODO: Consider subsequent query to determine which rows need to be updated then only update those attachments + List parents = new ArrayList<>(); + for (ListRecord record : records) + parents.add(new ListItemAttachmentParent(record.entityId, sourceContainer)); + + try + { + AttachmentService.get().moveAttachments(targetContainer, parents, user); + } + catch (IOException e) + { + errors.addRowError(new ValidationException("Failed to move attachments when moving list rows. Error: " + e.getMessage())); + } + } + + private int addDetailedMoveAuditEvents(User user, Container sourceContainer, Container targetContainer, List records) + { + List auditEvents = new ArrayList<>(records.size()); + String keyName = _list.getKeyName(); + String sourcePath = sourceContainer.getPath(); + String targetPath = targetContainer.getPath(); + + for (ListRecord record : records) + { + ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(targetContainer, "An existing list record was moved", _list); + event.setListItemEntityId(record.entityId); + event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(CaseInsensitiveHashMap.of("Folder", sourcePath, keyName, record.key.toString()))); + event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(CaseInsensitiveHashMap.of("Folder", targetPath, keyName, record.key.toString()))); + auditEvents.add(event); + } + + AuditLogService.get().addEvents(user, auditEvents, true); + + return auditEvents.size(); + } + + @Override + protected Map deleteRow(User user, Container container, Map oldRowMap) throws InvalidKeyException, QueryUpdateServiceException, SQLException + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to delete data from this table."); + + Map result = super.deleteRow(getListUser(user, container), container, oldRowMap); + + if (null != result) + { + String entityId = (String) result.get(ID); + + if (null != entityId) + { + ListManager mgr = ListManager.get(); + String deletedRecord = mgr.formatAuditItem(_list, user, result); + + // Audit + mgr.addAuditEvent(_list, user, container, "An existing list record was deleted", entityId, deletedRecord, null); + + // Remove discussions + if (DiscussionService.get() != null) + DiscussionService.get().deleteDiscussions(container, user, entityId); + + // Remove attachments + if (hasAttachmentProperties()) + AttachmentService.get().deleteAttachments(new ListItemAttachmentParent(entityId, container)); + + // Clean up Search indexer + if (!result.isEmpty()) + mgr.deleteItemIndex(_list, entityId); + } + } + + return result; + } + + + // Deletes attachments & discussions, and removes list documents from full-text search index. + public void deleteRelatedListData(final User user, final Container container) + { + // Unindex all item docs and the entire list doc + ListManager.get().deleteIndexedList(_list); + + // Delete attachments and discussions associated with a list in batches of 1,000 + new TableSelector(getDbTable(), Collections.singleton("entityId")).forEachBatch(String.class, 1000, new ForEachBatchBlock<>() + { + @Override + public boolean accept(String entityId) + { + return null != entityId; + } + + @Override + public void exec(List entityIds) + { + // delete the related list data for this block + deleteRelatedListData(user, container, entityIds); + } + }); + } + + // delete the related list data for this block of entityIds + private void deleteRelatedListData(User user, Container container, List entityIds) + { + // Build up set of entityIds and AttachmentParents + List attachmentParents = new ArrayList<>(); + + // Delete Discussions + if (_list.getDiscussionSetting() != ListDefinition.DiscussionSetting.None && DiscussionService.get() != null) + DiscussionService.get().deleteDiscussions(container, user, entityIds); + + // Delete Attachments + if (hasAttachmentProperties()) + { + for (String entityId : entityIds) + { + attachmentParents.add(new ListItemAttachmentParent(entityId, container)); + } + AttachmentService.get().deleteAttachments(attachmentParents); + } + } + + @Override + protected int truncateRows(User user, Container container) throws QueryUpdateServiceException, SQLException + { + int result; + try (DbScope.Transaction transaction = getDbTable().getSchema().getScope().ensureTransaction()) + { + deleteRelatedListData(user, container); + result = super.truncateRows(getListUser(user, container), container); + transaction.addCommitTask(() -> ListManager.get().addAuditEvent(_list, user, "Deleted " + result + " rows from list."), DbScope.CommitTaskOption.POSTCOMMIT); + transaction.commit(); + } + + return result; + } + + @Nullable + public SimpleFilter getKeyFilter(Map map) throws InvalidKeyException + { + String keyName = _list.getKeyName(); + Object key = getField(map, keyName); + + if (null == key) + { + // Auto-increment lists might not provide a key so allow them to pass through + if (ListDefinition.KeyType.AutoIncrementInteger.equals(_list.getKeyType())) + return null; + throw new InvalidKeyException("No " + keyName + " provided for list \"" + _list.getName() + "\""); + } + + return new SimpleFilter(FieldKey.fromParts(keyName), getKeyFilterValue(key)); + } + + @NotNull + private Object getKeyFilterValue(@NotNull Object key) + { + ListDefinition.KeyType type = _list.getKeyType(); + + // Check the type of the list to ensure proper casting of the key type + if (ListDefinition.KeyType.Integer.equals(type) || ListDefinition.KeyType.AutoIncrementInteger.equals(type)) + return isIntegral(key) ? key : Integer.valueOf(key.toString()); + + return key.toString(); + } + + @Nullable + private Object getField(Map map, String key) + { + /* TODO: this is very strange, we have a TableInfo we should be using its ColumnInfo objects to figure out aliases, we don't need to guess */ + Object value = map.get(key); + + if (null == value) + value = map.get(key + "_"); + + if (null == value) + value = map.get(getDbTable().getSqlDialect().legalNameFromName(key)); + + return value; + } + + /** + * Delegate class to generate an AttachmentParent + */ + public static class ListItemAttachmentParentFactory implements AttachmentParentFactory + { + @Override + public AttachmentParent generateAttachmentParent(String entityId, Container c) + { + return new ListItemAttachmentParent(entityId, c); + } + } + + /** + * Get Domain from list definition, unless null then get from super + */ + @Override + protected Domain getDomain() + { + return _list != null? + _list.getDomain() : + super.getDomain(); + } +} diff --git a/query/src/org/labkey/query/QueryServiceImpl.java b/query/src/org/labkey/query/QueryServiceImpl.java index c094ca989ce..d5a2c41447d 100644 --- a/query/src/org/labkey/query/QueryServiceImpl.java +++ b/query/src/org/labkey/query/QueryServiceImpl.java @@ -3098,7 +3098,7 @@ public void clearEnvironment() } @Override - public int moveAuditEvents(Container targetContainer, List rowPks, String schemaName, String queryName) + public int moveAuditEvents(Container targetContainer, List rowPks, String schemaName, String queryName) { QueryUpdateAuditProvider provider = new QueryUpdateAuditProvider(); return provider.moveEvents(targetContainer, rowPks, schemaName, queryName); diff --git a/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java b/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java index 77e37f019e9..2de865a1aca 100644 --- a/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java +++ b/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java @@ -1,311 +1,311 @@ -/* - * Copyright (c) 2013-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.query.audit; - -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.AuditTypeProvider; -import org.labkey.api.audit.DetailedAuditTypeEvent; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.query.AbstractAuditDomainKind; -import org.labkey.api.audit.query.DefaultAuditTypeTable; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.MutableColumnInfo; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.TableInfo; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryForm; -import org.labkey.api.query.QuerySettings; -import org.labkey.api.query.QueryView; -import org.labkey.api.query.UserSchema; -import org.labkey.api.view.ViewContext; -import org.labkey.query.controllers.QueryController; -import org.springframework.validation.BindException; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * User: klum - * Date: 7/21/13 - */ -public class QueryUpdateAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider -{ - public static final String QUERY_UPDATE_AUDIT_EVENT = "QueryUpdateAuditEvent"; - - public static final String COLUMN_NAME_ROW_PK = "RowPk"; - public static final String COLUMN_NAME_SCHEMA_NAME = "SchemaName"; - public static final String COLUMN_NAME_QUERY_NAME = "QueryName"; - - static final List defaultVisibleColumns = new ArrayList<>(); - - static { - - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED_BY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_IMPERSONATED_BY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_PROJECT_ID)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CONTAINER)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_SCHEMA_NAME)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_QUERY_NAME)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_COMMENT)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_USER_COMMENT)); - } - - public QueryUpdateAuditProvider() - { - super(new QueryUpdateAuditDomainKind()); - } - - @Override - public String getEventName() - { - return QUERY_UPDATE_AUDIT_EVENT; - } - - @Override - public String getLabel() - { - return "Query update events"; - } - - @Override - public String getDescription() - { - return "Data about insert and update queries."; - } - - @Override - public Map legacyNameMap() - { - Map legacyMap = super.legacyNameMap(); - legacyMap.put(FieldKey.fromParts("key1"), COLUMN_NAME_ROW_PK); - legacyMap.put(FieldKey.fromParts("key2"), COLUMN_NAME_SCHEMA_NAME); - legacyMap.put(FieldKey.fromParts("key3"), COLUMN_NAME_QUERY_NAME); - legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.OLD_RECORD_PROP_NAME), AbstractAuditDomainKind.OLD_RECORD_PROP_NAME); - legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.NEW_RECORD_PROP_NAME), AbstractAuditDomainKind.NEW_RECORD_PROP_NAME); - return legacyMap; - } - - @Override - public Class getEventClass() - { - return (Class)QueryUpdateAuditEvent.class; - } - - @Override - public TableInfo createTableInfo(UserSchema userSchema, ContainerFilter cf) - { - DefaultAuditTypeTable table = new DefaultAuditTypeTable(this, createStorageTableInfo(), userSchema, cf, defaultVisibleColumns) - { - @Override - protected void initColumn(MutableColumnInfo col) - { - if (COLUMN_NAME_SCHEMA_NAME.equalsIgnoreCase(col.getName())) - { - col.setLabel("Schema Name"); - } - else if (COLUMN_NAME_QUERY_NAME.equalsIgnoreCase(col.getName())) - { - col.setLabel("Query Name"); - } - else if (COLUMN_NAME_USER_COMMENT.equalsIgnoreCase(col.getName())) - { - col.setLabel("User Comment"); - } - } - }; - appendValueMapColumns(table, QUERY_UPDATE_AUDIT_EVENT); - - return table; - } - - @Override - public List getDefaultVisibleColumns() - { - return defaultVisibleColumns; - } - - - public static QueryView createHistoryQueryView(ViewContext context, QueryForm form, BindException errors) - { - UserSchema schema = AuditLogService.getAuditLogSchema(context.getUser(), context.getContainer()); - if (schema != null) - { - QuerySettings settings = new QuerySettings(context, QueryView.DATAREGIONNAME_DEFAULT); - - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_SCHEMA_NAME), form.getSchemaName()); - filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_QUERY_NAME), form.getQueryName()); - - settings.setBaseFilter(filter); - settings.setQueryName(QUERY_UPDATE_AUDIT_EVENT); - return schema.createView(context, settings, errors); - } - return null; - } - - public static QueryView createDetailsQueryView(ViewContext context, QueryController.QueryDetailsForm form, BindException errors) - { - return createDetailsQueryView(context, form.getSchemaName(), form.getQueryName(), form.getKeyValue(), errors); - } - - public static QueryView createDetailsQueryView(ViewContext context, String schemaName, String queryName, String keyValue, BindException errors) - { - UserSchema schema = AuditLogService.getAuditLogSchema(context.getUser(), context.getContainer()); - if (schema != null) - { - QuerySettings settings = new QuerySettings(context, QueryView.DATAREGIONNAME_DEFAULT); - - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_SCHEMA_NAME), schemaName); - filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_QUERY_NAME), queryName); - filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_ROW_PK), keyValue); - - settings.setBaseFilter(filter); - settings.setQueryName(QUERY_UPDATE_AUDIT_EVENT); - return schema.createView(context, settings, errors); - } - return null; - } - - public int moveEvents(Container targetContainer, Collection rowIds, String schemaName, String queryName) - { - TableInfo auditTable = createStorageTableInfo(); - SQLFragment sql = new SQLFragment("UPDATE ").append(auditTable) - .append(" SET container = ").appendValue(targetContainer) - .append(" WHERE RowPk "); - auditTable.getSchema().getSqlDialect().appendInClauseSql(sql, rowIds.stream().map(Object::toString).toList()); - sql.append(" AND SchemaName = ").appendValue(schemaName).append(" AND QueryName = ").appendValue(queryName); - return new SqlExecutor(auditTable.getSchema()).execute(sql); - } - - public static class QueryUpdateAuditEvent extends DetailedAuditTypeEvent - { - private String _rowPk; - private String _schemaName; - private String _queryName; - - /** Important for reflection-based instantiation */ - @SuppressWarnings("unused") - public QueryUpdateAuditEvent() - { - super(); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); - } - - public QueryUpdateAuditEvent(Container container, String comment) - { - super(QUERY_UPDATE_AUDIT_EVENT, container, comment); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); - } - - public String getRowPk() - { - return _rowPk; - } - - public void setRowPk(String rowPk) - { - _rowPk = rowPk; - } - - public String getSchemaName() - { - return _schemaName; - } - - public void setSchemaName(String schemaName) - { - _schemaName = schemaName; - } - - public String getQueryName() - { - return _queryName; - } - - public void setQueryName(String queryName) - { - _queryName = queryName; - } - - @Override - public Map getAuditLogMessageElements() - { - Map elements = new LinkedHashMap<>(); - elements.put("rowPk", getRowPk()); - elements.put("schemaName", getSchemaName()); - elements.put("queryName", getQueryName()); - elements.put("transactionId", getTransactionId()); - elements.put("userComment", getUserComment()); - // N.B. oldRecordMap and newRecordMap are potentially very large (and are not displayed in the default grid view) - elements.putAll(super.getAuditLogMessageElements()); - return elements; - } - } - - public static class QueryUpdateAuditDomainKind extends AbstractAuditDomainKind - { - public static final String NAME = "QueryUpdateAuditDomain"; - public static String NAMESPACE_PREFIX = "Audit-" + NAME; - - private final Set _fields; - - public QueryUpdateAuditDomainKind() - { - super(QUERY_UPDATE_AUDIT_EVENT); - - Set fields = new LinkedHashSet<>(); - fields.add(createPropertyDescriptor(COLUMN_NAME_ROW_PK, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_SCHEMA_NAME, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_QUERY_NAME, PropertyType.STRING)); - fields.add(createOldDataMapPropertyDescriptor()); - fields.add(createNewDataMapPropertyDescriptor()); - fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_ID, PropertyType.BIGINT)); - fields.add(createPropertyDescriptor(COLUMN_NAME_USER_COMMENT, PropertyType.STRING)); - _fields = Collections.unmodifiableSet(fields); - } - - @Override - public Set getProperties() - { - return _fields; - } - - @Override - protected String getNamespacePrefix() - { - return NAMESPACE_PREFIX; - } - - @Override - public String getKindName() - { - return NAME; - } - } -} +/* + * Copyright (c) 2013-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.query.audit; + +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.AuditTypeProvider; +import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.query.AbstractAuditDomainKind; +import org.labkey.api.audit.query.DefaultAuditTypeTable; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.MutableColumnInfo; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.TableInfo; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryForm; +import org.labkey.api.query.QuerySettings; +import org.labkey.api.query.QueryView; +import org.labkey.api.query.UserSchema; +import org.labkey.api.view.ViewContext; +import org.labkey.query.controllers.QueryController; +import org.springframework.validation.BindException; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * User: klum + * Date: 7/21/13 + */ +public class QueryUpdateAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider +{ + public static final String QUERY_UPDATE_AUDIT_EVENT = "QueryUpdateAuditEvent"; + + public static final String COLUMN_NAME_ROW_PK = "RowPk"; + public static final String COLUMN_NAME_SCHEMA_NAME = "SchemaName"; + public static final String COLUMN_NAME_QUERY_NAME = "QueryName"; + + static final List defaultVisibleColumns = new ArrayList<>(); + + static { + + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED_BY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_IMPERSONATED_BY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_PROJECT_ID)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CONTAINER)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_SCHEMA_NAME)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_QUERY_NAME)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_COMMENT)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_USER_COMMENT)); + } + + public QueryUpdateAuditProvider() + { + super(new QueryUpdateAuditDomainKind()); + } + + @Override + public String getEventName() + { + return QUERY_UPDATE_AUDIT_EVENT; + } + + @Override + public String getLabel() + { + return "Query update events"; + } + + @Override + public String getDescription() + { + return "Data about insert and update queries."; + } + + @Override + public Map legacyNameMap() + { + Map legacyMap = super.legacyNameMap(); + legacyMap.put(FieldKey.fromParts("key1"), COLUMN_NAME_ROW_PK); + legacyMap.put(FieldKey.fromParts("key2"), COLUMN_NAME_SCHEMA_NAME); + legacyMap.put(FieldKey.fromParts("key3"), COLUMN_NAME_QUERY_NAME); + legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.OLD_RECORD_PROP_NAME), AbstractAuditDomainKind.OLD_RECORD_PROP_NAME); + legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.NEW_RECORD_PROP_NAME), AbstractAuditDomainKind.NEW_RECORD_PROP_NAME); + return legacyMap; + } + + @Override + public Class getEventClass() + { + return (Class)QueryUpdateAuditEvent.class; + } + + @Override + public TableInfo createTableInfo(UserSchema userSchema, ContainerFilter cf) + { + DefaultAuditTypeTable table = new DefaultAuditTypeTable(this, createStorageTableInfo(), userSchema, cf, defaultVisibleColumns) + { + @Override + protected void initColumn(MutableColumnInfo col) + { + if (COLUMN_NAME_SCHEMA_NAME.equalsIgnoreCase(col.getName())) + { + col.setLabel("Schema Name"); + } + else if (COLUMN_NAME_QUERY_NAME.equalsIgnoreCase(col.getName())) + { + col.setLabel("Query Name"); + } + else if (COLUMN_NAME_USER_COMMENT.equalsIgnoreCase(col.getName())) + { + col.setLabel("User Comment"); + } + } + }; + appendValueMapColumns(table, QUERY_UPDATE_AUDIT_EVENT); + + return table; + } + + @Override + public List getDefaultVisibleColumns() + { + return defaultVisibleColumns; + } + + + public static QueryView createHistoryQueryView(ViewContext context, QueryForm form, BindException errors) + { + UserSchema schema = AuditLogService.getAuditLogSchema(context.getUser(), context.getContainer()); + if (schema != null) + { + QuerySettings settings = new QuerySettings(context, QueryView.DATAREGIONNAME_DEFAULT); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_SCHEMA_NAME), form.getSchemaName()); + filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_QUERY_NAME), form.getQueryName()); + + settings.setBaseFilter(filter); + settings.setQueryName(QUERY_UPDATE_AUDIT_EVENT); + return schema.createView(context, settings, errors); + } + return null; + } + + public static QueryView createDetailsQueryView(ViewContext context, QueryController.QueryDetailsForm form, BindException errors) + { + return createDetailsQueryView(context, form.getSchemaName(), form.getQueryName(), form.getKeyValue(), errors); + } + + public static QueryView createDetailsQueryView(ViewContext context, String schemaName, String queryName, String keyValue, BindException errors) + { + UserSchema schema = AuditLogService.getAuditLogSchema(context.getUser(), context.getContainer()); + if (schema != null) + { + QuerySettings settings = new QuerySettings(context, QueryView.DATAREGIONNAME_DEFAULT); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_SCHEMA_NAME), schemaName); + filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_QUERY_NAME), queryName); + filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_ROW_PK), keyValue); + + settings.setBaseFilter(filter); + settings.setQueryName(QUERY_UPDATE_AUDIT_EVENT); + return schema.createView(context, settings, errors); + } + return null; + } + + public int moveEvents(Container targetContainer, Collection rowPks, String schemaName, String queryName) + { + TableInfo auditTable = createStorageTableInfo(); + SQLFragment sql = new SQLFragment("UPDATE ").append(auditTable) + .append(" SET container = ").appendValue(targetContainer) + .append(" WHERE RowPk "); + auditTable.getSchema().getSqlDialect().appendInClauseSql(sql, rowPks.stream().map(Object::toString).toList()); + sql.append(" AND SchemaName = ").appendValue(schemaName).append(" AND QueryName = ").appendValue(queryName); + return new SqlExecutor(auditTable.getSchema()).execute(sql); + } + + public static class QueryUpdateAuditEvent extends DetailedAuditTypeEvent + { + private String _rowPk; + private String _schemaName; + private String _queryName; + + /** Important for reflection-based instantiation */ + @SuppressWarnings("unused") + public QueryUpdateAuditEvent() + { + super(); + setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); + } + + public QueryUpdateAuditEvent(Container container, String comment) + { + super(QUERY_UPDATE_AUDIT_EVENT, container, comment); + setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); + } + + public String getRowPk() + { + return _rowPk; + } + + public void setRowPk(String rowPk) + { + _rowPk = rowPk; + } + + public String getSchemaName() + { + return _schemaName; + } + + public void setSchemaName(String schemaName) + { + _schemaName = schemaName; + } + + public String getQueryName() + { + return _queryName; + } + + public void setQueryName(String queryName) + { + _queryName = queryName; + } + + @Override + public Map getAuditLogMessageElements() + { + Map elements = new LinkedHashMap<>(); + elements.put("rowPk", getRowPk()); + elements.put("schemaName", getSchemaName()); + elements.put("queryName", getQueryName()); + elements.put("transactionId", getTransactionId()); + elements.put("userComment", getUserComment()); + // N.B. oldRecordMap and newRecordMap are potentially very large (and are not displayed in the default grid view) + elements.putAll(super.getAuditLogMessageElements()); + return elements; + } + } + + public static class QueryUpdateAuditDomainKind extends AbstractAuditDomainKind + { + public static final String NAME = "QueryUpdateAuditDomain"; + public static String NAMESPACE_PREFIX = "Audit-" + NAME; + + private final Set _fields; + + public QueryUpdateAuditDomainKind() + { + super(QUERY_UPDATE_AUDIT_EVENT); + + Set fields = new LinkedHashSet<>(); + fields.add(createPropertyDescriptor(COLUMN_NAME_ROW_PK, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_SCHEMA_NAME, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_QUERY_NAME, PropertyType.STRING)); + fields.add(createOldDataMapPropertyDescriptor()); + fields.add(createNewDataMapPropertyDescriptor()); + fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_ID, PropertyType.BIGINT)); + fields.add(createPropertyDescriptor(COLUMN_NAME_USER_COMMENT, PropertyType.STRING)); + _fields = Collections.unmodifiableSet(fields); + } + + @Override + public Set getProperties() + { + return _fields; + } + + @Override + protected String getNamespacePrefix() + { + return NAMESPACE_PREFIX; + } + + @Override + public String getKindName() + { + return NAME; + } + } +} From 47e751fbf9373ee4644f9942a8ef444175f62986 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Tue, 30 Sep 2025 10:56:50 -0700 Subject: [PATCH 02/11] CRLF --- .../org/labkey/api/data/ContainerManager.java | 6274 ++++++++--------- .../experiment/api/SampleTypeServiceImpl.java | 2 +- .../list/model/ListQueryUpdateService.java | 1685 ++--- .../query/audit/QueryUpdateAuditProvider.java | 618 +- 4 files changed, 4289 insertions(+), 4290 deletions(-) diff --git a/api/src/org/labkey/api/data/ContainerManager.java b/api/src/org/labkey/api/data/ContainerManager.java index 415810daa51..bef6d394da8 100644 --- a/api/src/org/labkey/api/data/ContainerManager.java +++ b/api/src/org/labkey/api/data/ContainerManager.java @@ -1,3137 +1,3137 @@ -/* - * Copyright (c) 2005-2018 Fred Hutchinson Cancer Research Center - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.data; - -import com.google.common.base.Enums; -import org.apache.commons.collections4.MultiValuedMap; -import org.apache.commons.collections4.multimap.ArrayListValuedHashMap; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.xmlbeans.XmlObject; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.labkey.api.Constants; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.admin.FolderExportContext; -import org.labkey.api.admin.FolderImportContext; -import org.labkey.api.admin.FolderImporterImpl; -import org.labkey.api.admin.FolderWriterImpl; -import org.labkey.api.admin.StaticLoggerGetter; -import org.labkey.api.attachments.AttachmentParent; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.provider.ContainerAuditProvider; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.ConcurrentHashSet; -import org.labkey.api.collections.IntHashMap; -import org.labkey.api.data.Container.ContainerException; -import org.labkey.api.data.Container.LockState; -import org.labkey.api.data.PropertyManager.WritablePropertyMap; -import org.labkey.api.data.SimpleFilter.InClause; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.data.validator.ColumnValidators; -import org.labkey.api.event.PropertyChange; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.module.FolderType; -import org.labkey.api.module.FolderTypeManager; -import org.labkey.api.module.Module; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.portal.ProjectUrls; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.SimpleValidationError; -import org.labkey.api.query.ValidationException; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.Group; -import org.labkey.api.security.MutableSecurityPolicy; -import org.labkey.api.security.SecurityLogger; -import org.labkey.api.security.SecurityManager; -import org.labkey.api.security.SecurityPolicyManager; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.CreateProjectPermission; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.InsertPermission; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.security.roles.AuthorRole; -import org.labkey.api.security.roles.ReaderRole; -import org.labkey.api.security.roles.Role; -import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.settings.AppProps; -import org.labkey.api.test.TestTimeout; -import org.labkey.api.test.TestWhen; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JunitUtil; -import org.labkey.api.util.MinorConfigurationException; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Path; -import org.labkey.api.util.QuietCloser; -import org.labkey.api.util.ReentrantLockWithName; -import org.labkey.api.util.ResultSetUtil; -import org.labkey.api.util.TestContext; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.FolderTab; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.NavTreeManager; -import org.labkey.api.view.Portal; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.view.ViewContext; -import org.labkey.api.writer.MemoryVirtualFile; -import org.labkey.folder.xml.FolderDocument; -import org.labkey.remoteapi.collections.CaseInsensitiveHashMap; -import org.springframework.validation.BindException; -import org.springframework.validation.Errors; - -import java.beans.PropertyChangeEvent; -import java.beans.PropertyChangeListener; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.ListIterator; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -import static org.labkey.api.action.SpringActionController.ERROR_GENERIC; - -/** - * This class manages a hierarchy of collections, backed by a database table called Containers. - * Containers are named using filesystem-like paths e.g. /proteomics/comet/. Each path - * maps to a UID and set of permissions. The current security scheme allows ACLs - * to be specified explicitly on the directory or completely inherited. ACLs are not combined. - *

- * NOTE: we act like java.io.File(). Paths start with forward-slash, but do not end with forward-slash. - * The root container's name is '/'. This means that it is not always the case that - * me.getPath() == me.getParent().getPath() + "/" + me.getName() - *

- * The synchronization goals are to keep invalid containers from creeping into the cache. For example, once - * a container is deleted, it should never get put back in the cache. We accomplish this by synchronizing on - * the removal from the cache, and the database lookup/cache insertion. While a container is in the middle - * of being deleted, it's OK for other clients to see it because FKs enforce that it's always internally - * consistent, even if some of the data has already been deleted. - */ -public class ContainerManager -{ - private static final Logger LOG = LogHelper.getLogger(ContainerManager.class, "Container (projects, folders, and workbooks) retrieval and management"); - private static final CoreSchema CORE = CoreSchema.getInstance(); - - private static final String PROJECT_LIST_ID = "Projects"; - - public static final String HOME_PROJECT_PATH = "/home"; - public static final String DEFAULT_SUPPORT_PROJECT_PATH = HOME_PROJECT_PATH + "/support"; - - private static final Cache CACHE_PATH = CacheManager.getCache(Constants.getMaxContainers(), CacheManager.DAY, "Containers by Path"); - private static final Cache CACHE_ENTITY_ID = CacheManager.getCache(Constants.getMaxContainers(), CacheManager.DAY, "Containers by EntityId"); - private static final Cache> CACHE_CHILDREN = CacheManager.getCache(Constants.getMaxContainers(), CacheManager.DAY, "Child EntityIds of Containers"); - private static final ReentrantLock DATABASE_QUERY_LOCK = new ReentrantLockWithName(ContainerManager.class, "DATABASE_QUERY_LOCK"); - public static final String FOLDER_TYPE_PROPERTY_SET_NAME = "folderType"; - public static final String FOLDER_TYPE_PROPERTY_NAME = "name"; - public static final String FOLDER_TYPE_PROPERTY_TABTYPE_OVERRIDDEN = "ctFolderTypeOverridden"; - public static final String TABFOLDER_CHILDREN_DELETED = "tabChildrenDeleted"; - public static final String AUDIT_SETTINGS_PROPERTY_SET_NAME = "containerAuditSettings"; - public static final String REQUIRE_USER_COMMENTS_PROPERTY_NAME = "requireUserComments"; - - private static final List _resourceProviders = new CopyOnWriteArrayList<>(); - - // containers that are being constructed, used to suppress events before fireCreateContainer() - private static final Set _constructing = new ConcurrentHashSet<>(); - - - /** enum of properties you can see in property change events */ - public enum Property - { - Name, - Parent, - Policy, - /** The default or active set of modules in the container has changed */ - Modules, - FolderType, - WebRoot, - AttachmentDirectory, - PipelineRoot, - Title, - Description, - SiteRoot, - StudyChange, - EndpointDirectory, - CloudStores - } - - static Path makePath(Container parent, String name) - { - if (null == parent) - return new Path(name); - return parent.getParsedPath().append(name, true); - } - - public static Container createMockContainer() - { - return new Container(null, "MockContainer", "01234567-8901-2345-6789-012345678901", 99999999, 0, new Date(), User.guest.getUserId(), true); - } - - private static Container createRoot() - { - Map m = new HashMap<>(); - m.put("Parent", null); - m.put("Name", ""); - Table.insert(null, CORE.getTableInfoContainers(), m); - - return getRoot(); - } - - private static DbScope.Transaction ensureTransaction() - { - return CORE.getSchema().getScope().ensureTransaction(DATABASE_QUERY_LOCK); - } - - private static int getNewChildSortOrder(Container parent) - { - int nextSortOrderVal = 0; - - List children = parent.getChildren(); - if (children != null) - { - for (Container child : children) - { - // find the max sort order value for the set of children - nextSortOrderVal = Math.max(nextSortOrderVal, child.getSortOrder()); - } - } - - // custom sorting applies: put new container at the end. - if (nextSortOrderVal > 0) - return nextSortOrderVal + 1; - - // we're sorted alphabetically - return 0; - } - - // TODO: Make private and force callers to use ensureContainer instead? - // TODO: Handle root creation here? - @NotNull - public static Container createContainer(Container parent, String name, @NotNull User user) - { - return createContainer(parent, name, null, null, NormalContainerType.NAME, user, null, null); - } - - public static final String WORKBOOK_DBSEQUENCE_NAME = "org.labkey.api.data.Workbooks"; - - // TODO: Pass in FolderType (separate from the container type of workbook, etc) and transact it with container creation? - @NotNull - public static Container createContainer(Container parent, String name, @Nullable String title, @Nullable String description, String type, @NotNull User user) - { - return createContainer(parent, name, title, description, type, user, null, null); - } - - @NotNull - public static Container createContainer(Container parent, String name, @Nullable String title, @Nullable String description, String type, @NotNull User user, @Nullable String auditMsg) - { - return createContainer(parent, name, title, description, type, user, auditMsg, null); - } - - @NotNull - public static Container createContainer(Container parent, String name, @Nullable String title, @Nullable String description, String type, @NotNull User user, @Nullable String auditMsg, - Consumer configureContainer) - { - ContainerType cType = ContainerTypeRegistry.get().getType(type); - if (cType == null) - throw new IllegalArgumentException("Unknown container type: " + type); - - // TODO: move this to ContainerType? - long sortOrder; - if (cType instanceof WorkbookContainerType) - { - sortOrder = DbSequenceManager.get(parent, WORKBOOK_DBSEQUENCE_NAME).next(); - - // Default workbook names are simply "" - if (name == null) - name = String.valueOf(sortOrder); - } - else - { - sortOrder = getNewChildSortOrder(parent); - } - - if (!parent.canHaveChildren()) - throw new IllegalArgumentException("Parent of a container must not be a " + parent.getContainerType().getName()); - - StringBuilder error = new StringBuilder(); - if (!Container.isLegalName(name, parent.isRoot(), error)) - throw new ApiUsageException(error.toString()); - - if (!Container.isLegalTitle(title, error)) - throw new ApiUsageException(error.toString()); - - Path path = makePath(parent, name); - SQLException sqlx = null; - Map insertMap = null; - - GUID entityId = new GUID(); - Container c; - - try - { - _constructing.add(entityId); - - try - { - Map m = new CaseInsensitiveHashMap<>(); - m.put("Parent", parent.getId()); - m.put("Name", name); - m.put("Title", title); - m.put("SortOrder", sortOrder); - m.put("EntityId", entityId); - if (null != description) - m.put("Description", description); - m.put("Type", type); - insertMap = Table.insert(user, CORE.getTableInfoContainers(), m); - } - catch (RuntimeSQLException x) - { - if (!x.isConstraintException()) - throw x; - sqlx = x.getSQLException(); - } - - _clearChildrenFromCache(parent); - - c = insertMap == null ? null : getForId(entityId); - - if (null == c) - { - if (null != sqlx) - throw new RuntimeSQLException(sqlx); - else - throw new RuntimeException("Container for path '" + path + "' was not created properly."); - } - - User savePolicyUser = user; - if (c.isProject() && !c.hasPermission(user, AdminPermission.class) && ContainerManager.getRoot().hasPermission(user, CreateProjectPermission.class)) - { - // Special case for project creators who don't necessarily yet have permission to save the policy of - // the project they just created - savePolicyUser = User.getAdminServiceUser(); - } - - // Workbooks inherit perms from their parent so don't create a policy if this is a workbook - if (c.isContainerFor(ContainerType.DataType.permissions)) - { - SecurityManager.setAdminOnlyPermissions(c, savePolicyUser); - } - - _removeFromCache(c, true); // seems odd, but it removes c.getProject() which clears other things from the cache - - // Initialize the list of active modules in the Container - c.getActiveModules(true, true, user); - - if (c.isProject()) - { - SecurityManager.createNewProjectGroups(c, savePolicyUser); - } - else - { - // If current user does NOT have admin permission on this container or the project has been - // explicitly set to have new subfolders inherit permissions, then inherit permissions - // (otherwise they would not be able to see the folder) - boolean hasAdminPermission = c.hasPermission(user, AdminPermission.class); - if ((!hasAdminPermission && !user.hasRootAdminPermission()) || SecurityManager.shouldNewSubfoldersInheritPermissions(c.getProject())) - SecurityManager.setInheritPermissions(c); - } - - // NOTE parent caches some info about children (e.g. hasWorkbookChildren) - // since mutating cached objects is frowned upon, just uncache parent - // CONSIDER: we could perhaps only uncache if the child is a workbook, but I think this reasonable - _removeFromCache(parent, true); - - if (null != configureContainer) - configureContainer.accept(c); - } - finally - { - _constructing.remove(entityId); - } - - fireCreateContainer(c, user, auditMsg); - - return c; - } - - public static void addSecurableResourceProvider(ContainerSecurableResourceProvider provider) - { - _resourceProviders.add(provider); - } - - public static List getSecurableResourceProviders() - { - return Collections.unmodifiableList(_resourceProviders); - } - - public static Container createContainerFromTemplate(Container parent, String name, String title, Container templateContainer, User user, FolderExportContext exportCtx, Consumer afterCreateHandler) throws Exception - { - MemoryVirtualFile vf = new MemoryVirtualFile(); - - // export objects from the source template folder - FolderWriterImpl writer = new FolderWriterImpl(); - writer.write(templateContainer, exportCtx, vf); - - // create the new target container - Container c = createContainer(parent, name, title, null, NormalContainerType.NAME, user, null, afterCreateHandler); - - // import objects into the target folder - XmlObject folderXml = vf.getXmlBean("folder.xml"); - if (folderXml instanceof FolderDocument folderDoc) - { - FolderImportContext importCtx = new FolderImportContext(user, c, folderDoc, null, new StaticLoggerGetter(LogManager.getLogger(FolderImporterImpl.class)), vf); - - FolderImporterImpl importer = new FolderImporterImpl(); - importer.process(null, importCtx, vf); - } - - return c; - } - - public static void setRequireAuditComments(Container container, User user, @NotNull Boolean required) - { - WritablePropertyMap props = PropertyManager.getWritableProperties(container, AUDIT_SETTINGS_PROPERTY_SET_NAME, true); - String originalValue = props.get(REQUIRE_USER_COMMENTS_PROPERTY_NAME); - props.put(REQUIRE_USER_COMMENTS_PROPERTY_NAME, required.toString()); - props.save(); - - addAuditEvent(user, container, - "Changed " + REQUIRE_USER_COMMENTS_PROPERTY_NAME + " from \"" + - originalValue + "\" to \"" + required + "\""); - } - - public static void setFolderType(Container c, FolderType folderType, User user, BindException errors) - { - FolderType oldType = c.getFolderType(); - - if (folderType.equals(oldType)) - return; - - List errorStrings = new ArrayList<>(); - - if (!c.isProject() && folderType.isProjectOnlyType()) - errorStrings.add("Cannot set a subfolder to " + folderType.getName() + " because it is a project-only folder type."); - - // Check for any containers that need to be moved into container tabs - if (errorStrings.isEmpty() && folderType.hasContainerTabs()) - { - List childTabFoldersNonMatchingTypes = new ArrayList<>(); - List containersBecomingTabs = findAndCheckContainersMatchingTabs(c, folderType, childTabFoldersNonMatchingTypes, errorStrings); - - if (errorStrings.isEmpty()) - { - if (!containersBecomingTabs.isEmpty()) - { - // Make containers tab container; Folder tab will find them by name - try (DbScope.Transaction transaction = ensureTransaction()) - { - for (Container container : containersBecomingTabs) - updateType(container, TabContainerType.NAME, user); - - transaction.commit(); - } - } - - // Check these and change type unless they were overridden explicitly - for (Container container : childTabFoldersNonMatchingTypes) - { - if (!isContainerTabTypeOverridden(container)) - { - FolderTab newTab = folderType.findTab(container.getName()); - assert null != newTab; // There must be a tab because it caused the container to get into childTabFoldersNonMatchingTypes - FolderType newType = newTab.getFolderType(); - if (null == newType) - newType = FolderType.NONE; // default to NONE - setFolderType(container, newType, user, errors); - } - } - } - } - - if (errorStrings.isEmpty()) - { - oldType.unconfigureContainer(c, user); - WritablePropertyMap props = PropertyManager.getWritableProperties(c, FOLDER_TYPE_PROPERTY_SET_NAME, true); - props.put(FOLDER_TYPE_PROPERTY_NAME, folderType.getName()); - - if (c.isContainerTab()) - { - boolean containerTabTypeOverridden = false; - FolderTab tab = c.getParent().getFolderType().findTab(c.getName()); - if (null != tab && !folderType.equals(tab.getFolderType())) - containerTabTypeOverridden = true; - props.put(FOLDER_TYPE_PROPERTY_TABTYPE_OVERRIDDEN, Boolean.toString(containerTabTypeOverridden)); - } - props.save(); - - notifyContainerChange(c.getId(), Property.FolderType, user); - folderType.configureContainer(c, user); // Configure new only after folder type has been changed - - // TODO: Not needed? I don't think we've changed the container's state. - _removeFromCache(c, false); - } - else - { - for (String errorString : errorStrings) - errors.reject(SpringActionController.ERROR_MSG, errorString); - } - } - - public static void checkContainerValidity(Container c) throws ContainerException - { - // Check container for validity; in rare cases user may have changed their custom folderType.xml and caused - // duplicate subfolders (same name) to exist - // Get list of child containers that are not container tabs, but match container tabs; these are bad - FolderType folderType = getFolderType(c); - List errorStrings = new ArrayList<>(); - List childTabFoldersNonMatchingTypes = new ArrayList<>(); - List containersMatchingTabs = findAndCheckContainersMatchingTabs(c, folderType, childTabFoldersNonMatchingTypes, errorStrings); - if (!containersMatchingTabs.isEmpty()) - { - throw new Container.ContainerException("Folder " + c.getPath() + - " has a subfolder with the same name as a container tab folder, which is an invalid state." + - " This may have been caused by changing the folder type's tabs after this folder was set to its folder type." + - " An administrator should either delete the offending subfolder or change the folder's folder type.\n"); - } - } - - public static List findAndCheckContainersMatchingTabs(Container c, FolderType folderType, - List childTabFoldersNonMatchingTypes, List errorStrings) - { - List containersMatchingTabs = new ArrayList<>(); - for (FolderTab folderTab : folderType.getDefaultTabs()) - { - if (folderTab.getTabType() == FolderTab.TAB_TYPE.Container) - { - for (Container child : c.getChildren()) - { - if (child.getName().equalsIgnoreCase(folderTab.getName())) - { - if (!child.getFolderType().getName().equalsIgnoreCase(folderTab.getFolderTypeName())) - { - if (child.isContainerTab()) - childTabFoldersNonMatchingTypes.add(child); // Tab type doesn't match child tab folder - else - errorStrings.add("Child folder " + child.getName() + - " matches container tab, but folder type " + child.getFolderType().getName() + " doesn't match tab's folder type " + - folderTab.getFolderTypeName() + "."); - } - - int childCount = child.getChildren().size(); - if (childCount > 0) - { - errorStrings.add("Child folder " + child.getName() + - " matches container tab, but cannot be converted to a tab folder because it has " + childCount + " children."); - } - - if (!child.isConvertibleToTab()) - { - errorStrings.add("Child folder " + child.getName() + - " matches container tab, but cannot be converted to a tab folder because it is a " + child.getContainerNoun() + "."); - } - - if (!child.isContainerTab()) - containersMatchingTabs.add(child); - - break; // we found name match; can't be another - } - } - } - } - return containersMatchingTabs; - } - - private static final Set containersWithBadFolderTypes = new ConcurrentHashSet<>(); - - @NotNull - public static FolderType getFolderType(Container c) - { - String name = getFolderTypeName(c); - FolderType folderType; - - if (null != name) - { - folderType = FolderTypeManager.get().getFolderType(name); - - if (null == folderType) - { - // If we're upgrading then folder types won't be defined yet... don't warn in that case. - if (!ModuleLoader.getInstance().isUpgradeInProgress() && - !ModuleLoader.getInstance().isUpgradeRequired() && - !containersWithBadFolderTypes.contains(c)) - { - LOG.warn("No such folder type " + name + " for folder " + c.toString()); - containersWithBadFolderTypes.add(c); - } - - folderType = FolderType.NONE; - } - } - else - folderType = FolderType.NONE; - - return folderType; - } - - /** - * Most code should call getFolderType() instead. - * Useful for finding the name of the folder type BEFORE startup is complete, so the FolderType itself - * may not be available. - */ - @Nullable - public static String getFolderTypeName(Container c) - { - Map props = PropertyManager.getProperties(c, FOLDER_TYPE_PROPERTY_SET_NAME); - return props.get(FOLDER_TYPE_PROPERTY_NAME); - } - - - @NotNull - public static Map getFolderTypeNameContainerCounts(Container root) - { - Map nameCounts = new TreeMap<>(); - for (Container c : getAllChildren(root)) - { - Integer count = nameCounts.get(c.getFolderType().getName()); - if (null == count) - { - count = Integer.valueOf(0); - } - nameCounts.put(c.getFolderType().getName(), ++count); - } - return nameCounts; - } - - @NotNull - public static Map getProductFoldersMetrics(@NotNull FolderType folderType) - { - Container root = getRoot(); - Map metrics = new TreeMap<>(); - List counts = new ArrayList<>(); - for (Container c : root.getChildren()) - { - if (!c.getFolderType().getName().equals(folderType.getName())) - continue; - - int childCount = c.getChildren().stream().filter(Container::isInFolderNav).toList().size(); - counts.add(childCount); - } - - int totalFolderTypeMatch = counts.size(); - if (totalFolderTypeMatch == 0) - return metrics; - - Collections.sort(counts); - int median = counts.get((totalFolderTypeMatch - 1)/2); - if (totalFolderTypeMatch % 2 == 0 ) - { - int low = counts.get(totalFolderTypeMatch/2 - 1); - int high = counts.get(totalFolderTypeMatch/2); - median = Math.round((low + high) / 2.0f); - } - int maxProjectsCount = counts.get(totalFolderTypeMatch - 1); - int totalProjectsCount = counts.stream().mapToInt(Integer::intValue).sum(); - int averageProjectsCount = Math.round((float) totalProjectsCount /totalFolderTypeMatch); - - metrics.put("totalSubProjectsCount", totalProjectsCount); - metrics.put("averageSubProjectsPerHomeProject", averageProjectsCount); - metrics.put("medianSubProjectsCountPerHomeProject", median); - metrics.put("maxSubProjectsCountInHomeProject", maxProjectsCount); - - return metrics; - } - - public static boolean isContainerTabTypeThisOrChildrenOverridden(Container c) - { - if (isContainerTabTypeOverridden(c)) - return true; - if (c.getFolderType().hasContainerTabs()) - { - for (Container child : c.getChildren()) - { - if (child.isContainerTab() && isContainerTabTypeOverridden(child)) - return true; - } - } - return false; - } - - public static boolean isContainerTabTypeOverridden(Container c) - { - Map props = PropertyManager.getProperties(c, FOLDER_TYPE_PROPERTY_SET_NAME); - String overridden = props.get(FOLDER_TYPE_PROPERTY_TABTYPE_OVERRIDDEN); - return (null != overridden) && overridden.equalsIgnoreCase("true"); - } - - private static void setContainerTabDeleted(Container c, String tabName, String folderTypeName) - { - // Add prop in this category - WritablePropertyMap props = PropertyManager.getWritableProperties(c, TABFOLDER_CHILDREN_DELETED, true); - props.put(getDeletedTabKey(tabName, folderTypeName), "true"); - props.save(); - } - - public static void clearContainerTabDeleted(Container c, String tabName, String folderTypeName) - { - WritablePropertyMap props = PropertyManager.getWritableProperties(c, TABFOLDER_CHILDREN_DELETED, true); - String key = getDeletedTabKey(tabName, folderTypeName); - if (props.containsKey(key)) - { - props.remove(key); - props.save(); - } - } - - public static boolean hasContainerTabBeenDeleted(Container c, String tabName, String folderTypeName) - { - // We keep arbitrary number of deleted children tabs using suffix 0, 1, 2.... - Map props = PropertyManager.getProperties(c, TABFOLDER_CHILDREN_DELETED); - return props.containsKey(getDeletedTabKey(tabName, folderTypeName)); - } - - private static String getDeletedTabKey(String tabName, String folderTypeName) - { - return tabName + "-TABDELETED-FOLDER-" + folderTypeName; - } - - @NotNull - public static Container ensureContainer(@NotNull String path, @NotNull User user) - { - return ensureContainer(Path.parse(path), user); - } - - @NotNull - public static Container ensureContainer(@NotNull Path path, @NotNull User user) - { - Container c = null; - - try - { - c = getForPath(path); - } - catch (RootContainerException e) - { - // Ignore this -- root doesn't exist yet - } - - if (null == c) - { - if (path.isEmpty()) - c = createRoot(); - else - { - Path parentPath = path.getParent(); - c = ensureContainer(parentPath, user); - c = createContainer(c, path.getName(), null, null, NormalContainerType.NAME, user); - } - } - return c; - } - - - @NotNull - public static Container ensureContainer(Container parent, String name, User user) - { - // NOTE: Running outside a tx doesn't seem to be necessary. -// if (CORE.getSchema().getScope().isTransactionActive()) -// throw new IllegalStateException("Transaction should not be active"); - - Container c = null; - - try - { - c = getForPath(makePath(parent,name)); - } - catch (RootContainerException e) - { - // Ignore this -- root doesn't exist yet - } - - if (null == c) - { - c = createContainer(parent, name, user); - } - return c; - } - - public static void updateDescription(Container container, String description, User user) - throws ValidationException - { - ColumnValidators.validate(CORE.getTableInfoContainers().getColumn("Title"), null, 1, description); - - //For some reason, there is no primary key defined on core.containers - //so we can't use Table.update here - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - sql.append(" SET Description=? WHERE RowID=?").add(description).add(container.getRowId()); - new SqlExecutor(CORE.getSchema()).execute(sql); - - String oldValue = container.getDescription(); - _removeFromCache(container, false); - container = getForRowId(container.getRowId()); - ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(container, user, Property.Description, oldValue, description); - firePropertyChangeEvent(evt); - } - - public static void updateSearchable(Container container, boolean searchable, User user) - { - //For some reason, there is no primary key defined on core.containers - //so we can't use Table.update here - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - sql.append(" SET Searchable=? WHERE RowID=?").add(searchable).add(container.getRowId()); - new SqlExecutor(CORE.getSchema()).execute(sql); - - _removeFromCache(container, false); - } - - public static void updateLockState(Container container, LockState lockState, @NotNull Runnable auditRunnable) - { - //For some reason there is no primary key defined on core.containers - //so we can't use Table.update here - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - sql.append(" SET LockState = ?, ExpirationDate = NULL WHERE RowID = ?").add(lockState).add(container.getRowId()); - new SqlExecutor(CORE.getSchema()).execute(sql); - - _removeFromCache(container, false); - - auditRunnable.run(); - } - - public static List getExcludedProjects() - { - return getProjects().stream() - .filter(p->p.getLockState() == Container.LockState.Excluded) - .collect(Collectors.toList()); - } - - public static List getNonExcludedProjects() - { - return getProjects().stream() - .filter(p->p.getLockState() != Container.LockState.Excluded) - .collect(Collectors.toList()); - } - - public static void setExcludedProjects(Collection ids, @NotNull Runnable auditRunnable) - { - // First clear all existing "Excluded" states - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - sql.append(" SET LockState = NULL, ExpirationDate = NULL WHERE LockState = ?").add(LockState.Excluded); - new SqlExecutor(CORE.getSchema()).execute(sql); - - // Now set the passed-in projects to "Excluded" - if (!ids.isEmpty()) - { - ColumnInfo entityIdCol = CORE.getTableInfoContainers().getColumn("EntityId"); - Filter inClauseFilter = new SimpleFilter(new InClause(entityIdCol.getFieldKey(), ids)); - SQLFragment frag = new SQLFragment("UPDATE "); - frag.append(CORE.getTableInfoContainers().getSelectName()); - frag.append(" SET LockState = ?, ExpirationDate = NULL "); - frag.add(LockState.Excluded); - frag.append(inClauseFilter.getSQLFragment(CORE.getSqlDialect(), "c", Map.of(entityIdCol.getFieldKey(), entityIdCol))); - new SqlExecutor(CORE.getSchema()).execute(frag); - } - - clearCache(); - - auditRunnable.run(); - } - - public static void archiveContainer(User user, Container container, boolean archive) - { - if (container.isRoot() || container.isProject() || container.isAppHomeFolder()) - throw new ApiUsageException("Archive action not supported for this folder."); - - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers().getSelectName()); - if (archive) - { - sql.append(" SET LockState = ? "); - sql.add(LockState.Archived); - sql.append(" WHERE LockState IS NULL "); - } - else - { - sql.append(" SET LockState = NULL WHERE LockState = ? "); - sql.add(LockState.Archived); - } - sql.append("AND EntityId = ? "); - sql.add(container.getEntityId()); - new SqlExecutor(CORE.getSchema()).execute(sql); - - clearCache(); - - addAuditEvent(user, container, archive ? "Container has been archived." : "Archived container has been restored."); - } - - public static void updateExpirationDate(Container container, LocalDate expirationDate, @NotNull Runnable auditRunnable) - { - //For some reason there is no primary key defined on core.containers - //so we can't use Table.update here - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - // Note: jTDS doesn't support LocalDate, so convert to java.sql.Date - sql.append(" SET ExpirationDate = ? WHERE RowID = ?").add(java.sql.Date.valueOf(expirationDate)).add(container.getRowId()); - - new SqlExecutor(CORE.getSchema()).execute(sql); - - _removeFromCache(container, false); - - auditRunnable.run(); - } - - public static void updateType(Container container, String newType, User user) - { - //For some reason there is no primary key defined on core.containers - //so we can't use Table.update here - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - sql.append(" SET Type=? WHERE RowID=?").add(newType).add(container.getRowId()); - new SqlExecutor(CORE.getSchema()).execute(sql); - - _removeFromCache(container, false); - } - - public static void updateTitle(Container container, String title, User user) - throws ValidationException - { - ColumnValidators.validate(CORE.getTableInfoContainers().getColumn("Title"), null, 1, title); - - //For some reason there is no primary key defined on core.containers - //so we can't use Table.update here - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - sql.append(" SET Title=? WHERE RowID=?").add(title).add(container.getRowId()); - new SqlExecutor(CORE.getSchema()).execute(sql); - - _removeFromCache(container, false); - String oldValue = container.getTitle(); - container = getForRowId(container.getRowId()); - ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(container, user, Property.Title, oldValue, title); - firePropertyChangeEvent(evt); - } - - public static void uncache(Container c) - { - _removeFromCache(c, true); - } - - public static final String SHARED_CONTAINER_PATH = "/Shared"; - - @NotNull - public static Container getSharedContainer() - { - return ensureContainer(Path.parse(SHARED_CONTAINER_PATH), User.getAdminServiceUser()); - } - - public static List getChildren(Container parent) - { - return new ArrayList<>(getChildrenMap(parent).values()); - } - - // Default is to include all types of children, as seems only appropriate - public static List getChildren(Container parent, User u, Class perm) - { - return getChildren(parent, u, perm, null, ContainerTypeRegistry.get().getTypeNames()); - } - - public static List getChildren(Container parent, User u, Class perm, Set roles) - { - return getChildren(parent, u, perm, roles, ContainerTypeRegistry.get().getTypeNames()); - } - - public static List getChildren(Container parent, User u, Class perm, String typeIncluded) - { - return getChildren(parent, u, perm, null, Collections.singleton(typeIncluded)); - } - - public static List getChildren(Container parent, User u, Class perm, Set roles, Set includedTypes) - { - List children = new ArrayList<>(); - for (Container child : getChildrenMap(parent).values()) - if (includedTypes.contains(child.getContainerType().getName()) && child.hasPermission(u, perm, roles)) - children.add(child); - - return children; - } - - public static List getAllChildren(Container parent, User u) - { - return getAllChildren(parent, u, ReadPermission.class, null, ContainerTypeRegistry.get().getTypeNames()); - } - - public static List getAllChildren(Container parent, User u, Class perm) - { - return getAllChildren(parent, u, perm, null, ContainerTypeRegistry.get().getTypeNames()); - } - - // Default is to include all types of children - public static List getAllChildren(Container parent, User u, Class perm, Set roles) - { - return getAllChildren(parent, u, perm, roles, ContainerTypeRegistry.get().getTypeNames()); - } - - public static List getAllChildren(Container parent, User u, Class perm, String typeIncluded) - { - return getAllChildren(parent, u, perm, null, Collections.singleton(typeIncluded)); - } - - public static List getAllChildren(Container parent, User u, Class perm, Set roles, Set typesIncluded) - { - Set allChildren = getAllChildren(parent); - List result = new ArrayList<>(allChildren.size()); - - for (Container container : allChildren) - { - if (typesIncluded.contains(container.getContainerType().getName()) && container.hasPermission(u, perm, roles)) - { - result.add(container); - } - } - - return result; - } - - // Returns the next available child container name based on the baseName - public static String getAvailableChildContainerName(Container c, String baseName) - { - List children = getChildren(c); - Map folders = new HashMap<>(children.size() * 2); - for (Container child : children) - folders.put(child.getName(), child); - - String availableContainerName = baseName; - int i = 1; - while (folders.containsKey(availableContainerName)) - { - availableContainerName = baseName + " " + i++; - } - - return availableContainerName; - } - - // Returns true only if user has the specified permission in the entire container tree starting at root - public static boolean hasTreePermission(Container root, User u, Class perm) - { - for (Container c : getAllChildren(root)) - if (!c.hasPermission(u, perm)) - return false; - - return true; - } - - private static Map getChildrenMap(Container parent) - { - if (!parent.canHaveChildren()) - { - // Optimization to avoid database query (important because some installs have tens of thousands of - // workbooks) when the container is a workbook, which is not allowed to have children - return Collections.emptyMap(); - } - - List childIds = CACHE_CHILDREN.get(parent.getEntityId()); - if (null == childIds) - { - try (DbScope.Transaction t = ensureTransaction()) - { - List children = new SqlSelector(CORE.getSchema(), - "SELECT * FROM " + CORE.getTableInfoContainers() + " WHERE Parent = ? ORDER BY SortOrder, LOWER(Name)", - parent.getId()).getArrayList(Container.class); - - childIds = new ArrayList<>(children.size()); - for (Container c : children) - { - childIds.add(c.getEntityId()); - _addToCache(c); - } - childIds = Collections.unmodifiableList(childIds); - CACHE_CHILDREN.put(parent.getEntityId(), childIds); - // No database changes to commit, but need to decrement the transaction counter - t.commit(); - } - } - - if (childIds.isEmpty()) - return Collections.emptyMap(); - - // Use a LinkedHashMap to preserve the order defined by the user - they're not necessarily alphabetical - Map ret = new LinkedHashMap<>(); - for (GUID id : childIds) - { - Container c = getForId(id); - if (null != c) - ret.put(c.getName(), c); - } - return Collections.unmodifiableMap(ret); - } - - public static Container getForRowId(int id) - { - Selector selector = new SqlSelector(CORE.getSchema(), new SQLFragment("SELECT * FROM " + CORE.getTableInfoContainers() + " WHERE RowId = ?", id)); - return selector.getObject(Container.class); - } - - public static @Nullable Container getForId(@NotNull GUID guid) - { - return guid != null ? getForId(guid.toString()) : null; - } - - public static @Nullable Container getForId(@Nullable String id) - { - //if the input string is not a GUID, just return null, - //so that we don't get a SQLException when the database - //tries to convert it to a unique identifier. - if (!GUID.isGUID(id)) - return null; - - GUID guid = new GUID(id); - - Container d = CACHE_ENTITY_ID.get(guid); - if (null != d) - return d; - - try (DbScope.Transaction t = ensureTransaction()) - { - Container result = new SqlSelector( - CORE.getSchema(), - "SELECT * FROM " + CORE.getTableInfoContainers() + " WHERE EntityId = ?", - id).getObject(Container.class); - if (result != null) - { - result = _addToCache(result); - } - // No database changes to commit, but need to decrement the counter - t.commit(); - - return result; - } - } - - public static Container getChild(Container c, String name) - { - Path path = c.getParsedPath().append(name); - - Container d = _getFromCachePath(path); - if (null != d) - return d; - - Map map = getChildrenMap(c); - return map.get(name); - } - - - public static Container getForURL(@NotNull ActionURL url) - { - Container ret = getForPath(url.getExtraPath()); - if (ret == null) - ret = getForId(StringUtils.strip(url.getExtraPath(), "/")); - return ret; - } - - - public static Container getForPath(@NotNull String path) - { - if (GUID.isGUID(path)) - { - Container c = getForId(path); - if (c != null) - return c; - } - - Path p = Path.parse(path); - return getForPath(p); - } - - public static Container getForPath(Path path) - { - Container d = _getFromCachePath(path); - if (null != d) - return d; - - // Special case for ROOT -- we want to throw instead of returning null - if (path.equals(Path.rootPath)) - { - try (DbScope.Transaction t = ensureTransaction()) - { - TableInfo tinfo = CORE.getTableInfoContainers(); - - // Unusual, but possible -- if cache loader hits an exception it can end up caching null - if (null == tinfo) - throw new RootContainerException("Container table could not be retrieved from the cache"); - - // This might be called at bootstrap, before schemas have been created - if (tinfo.getTableType() == DatabaseTableType.NOT_IN_DB) - throw new RootContainerException("Container table has not been created"); - - Container result = new SqlSelector(CORE.getSchema(),"SELECT * FROM " + tinfo + " WHERE Parent IS NULL").getObject(Container.class); - - if (result == null) - throw new RootContainerException("Root container does not exist"); - - _addToCache(result); - // No database changes to commit, but need to decrement the counter - t.commit(); - return result; - } - } - else - { - Path parent = path.getParent(); - String name = path.getName(); - Container dirParent = getForPath(parent); - - if (null == dirParent) - return null; - - Map map = getChildrenMap(dirParent); - return map.get(name); - } - } - - public static class RootContainerException extends RuntimeException - { - private RootContainerException(String message, Throwable cause) - { - super(message, cause); - } - - private RootContainerException(String message) - { - super(message); - } - } - - public static Container getRoot() - { - try - { - return getForPath("/"); - } - catch (MinorConfigurationException e) - { - // If the server is misconfigured, rethrow so some callers don't swallow it and other callers don't end up - // reporting it to mothership, Issue 50843. - throw e; - } - catch (Exception e) - { - // Some callers catch and ignore this exception, e.g., early in the bootstrap process - throw new RootContainerException("Root container can't be retrieved", e); - } - } - - public static void saveAliasesForContainer(Container container, List aliases, User user) - { - Set originalAliases = new CaseInsensitiveHashSet(getAliasesForContainer(container)); - Set newAliases = new CaseInsensitiveHashSet(aliases); - - if (originalAliases.equals(newAliases)) - { - return; - } - - try (DbScope.Transaction transaction = ensureTransaction()) - { - // Delete all of the aliases for the current container, plus any of the aliases that might be associated - // with another container right now - SQLFragment deleteSQL = new SQLFragment(); - deleteSQL.append("DELETE FROM "); - deleteSQL.append(CORE.getTableInfoContainerAliases()); - deleteSQL.append(" WHERE ContainerRowId = ? "); - deleteSQL.add(container.getRowId()); - if (!aliases.isEmpty()) - { - deleteSQL.append(" OR Path IN ("); - String separator = ""; - for (String alias : aliases) - { - deleteSQL.append(separator); - separator = ", "; - deleteSQL.append("LOWER(?)"); - deleteSQL.add(alias); - } - deleteSQL.append(")"); - } - new SqlExecutor(CORE.getSchema()).execute(deleteSQL); - - // Store the alias as LOWER() so that we can query against it using the index - for (String alias : newAliases) - { - SQLFragment insertSQL = new SQLFragment(); - insertSQL.append("INSERT INTO "); - insertSQL.append(CORE.getTableInfoContainerAliases()); - insertSQL.append(" (Path, ContainerRowId) VALUES (LOWER(?), ?)"); - insertSQL.add(alias); - insertSQL.add(container.getRowId()); - new SqlExecutor(CORE.getSchema()).execute(insertSQL); - } - - addAuditEvent(user, container, - "Changed folder aliases from \"" + - StringUtils.join(originalAliases, ", ") + "\" to \"" + - StringUtils.join(newAliases, ", ") + "\""); - - transaction.commit(); - } - } - - // Abstract base class used for attaching system resources (favorite icons, logos, stylesheets, sso auth logos) to folders and projects - public static abstract class ContainerParent implements AttachmentParent - { - private final Container _c; - - protected ContainerParent(Container c) - { - _c = c; - } - - @Override - public String getEntityId() - { - return _c.getId(); - } - - @Override - public String getContainerId() - { - return _c.getId(); - } - - public Container getContainer() - { - return _c; - } - } - - public static Container getHomeContainer() - { - return getForPath(HOME_PROJECT_PATH); - } - - public static List getProjects() - { - return getChildren(getRoot()); - } - - public static NavTree getProjectList(ViewContext context, boolean includeChildren) - { - User user = context.getUser(); - Container currentProject = context.getContainer().getProject(); - String projectNavTreeId = PROJECT_LIST_ID; - if (currentProject != null) - projectNavTreeId += currentProject.getId(); - - NavTree navTree = (NavTree) NavTreeManager.getFromCache(projectNavTreeId, context); - if (null != navTree) - return navTree; - - NavTree list = new NavTree("Projects"); - List projects = getProjects(); - - for (Container project : projects) - { - boolean shouldDisplay = project.shouldDisplay(user) && project.hasPermission("getProjectList()", user, ReadPermission.class); - boolean includeCurrentProject = includeChildren && currentProject != null && currentProject.equals(project); - - if (shouldDisplay || includeCurrentProject) - { - ActionURL startURL = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(project); - - if (includeChildren) - list.addChild(getFolderListForUser(project, context)); - else if (project.equals(getHomeContainer())) - list.addChild(new NavTree("Home", startURL)); - else - list.addChild(project.getTitle(), startURL); - } - } - - list.setId(projectNavTreeId); - NavTreeManager.cacheTree(list, context.getUser()); - - return list; - } - - public static NavTree getFolderListForUser(final Container project, ViewContext viewContext) - { - final boolean isNavAccessOpen = AppProps.getInstance().isNavigationAccessOpen(); - final Container c = viewContext.getContainer(); - final String cacheKey = isNavAccessOpen ? project.getId() : c.getId(); - - NavTree tree = (NavTree) NavTreeManager.getFromCache(cacheKey, viewContext); - if (null != tree) - return tree; - - try - { - assert SecurityLogger.indent("getFolderListForUser()"); - - User user = viewContext.getUser(); - String projectId = project.getId(); - - List folders = new ArrayList<>(getAllChildren(project)); - - Collections.sort(folders); - - Set containersInTree = new HashSet<>(); - - Map m = new HashMap<>(); - Map permission = new HashMap<>(); - - for (Container f : folders) - { - if (!f.isInFolderNav()) - continue; - - boolean hasPolicyRead = f.hasPermission(user, ReadPermission.class); - - boolean skip = ( - !hasPolicyRead || - !f.shouldDisplay(user) || - !f.hasPermission(user, ReadPermission.class) - ); - - //Always put the project and current container in... - if (skip && !f.equals(project) && !f.equals(c)) - continue; - - //HACK to make home link consistent... - String name = f.getTitle(); - if (name.equals("home") && f.equals(getHomeContainer())) - name = "Home"; - - NavTree t = new NavTree(name); - - // 34137: Support folder path expansion for containers where label != name - t.setId(f.getId()); - if (hasPolicyRead) - { - ActionURL url = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(f); - t.setHref(url.getEncodedLocalURIString()); - } - - boolean addFolder = false; - - if (isNavAccessOpen) - { - addFolder = true; - } - else - { - // 32718: If navigation access is not open then hide projects that aren't directly - // accessible in site folder navigation. - - if (f.equals(c) || f.isRoot() || (hasPolicyRead && f.isProject())) - { - // In current container, root, or readable project - addFolder = true; - } - else - { - boolean isAscendant = f.isDescendant(c); - boolean isDescendant = c.isDescendant(f); - boolean inActivePath = isAscendant || isDescendant; - boolean hasAncestryRead = false; - - if (inActivePath) - { - Container leaf = isAscendant ? f : c; - Container localRoot = isAscendant ? c : f; - - List ancestors = containersToRootList(leaf); - Collections.reverse(ancestors); - - for (Container p : ancestors) - { - if (!permission.containsKey(p.getId())) - permission.put(p.getId(), p.hasPermission(user, ReadPermission.class)); - boolean hasRead = permission.get(p.getId()); - - if (p.equals(localRoot)) - { - hasAncestryRead = hasRead; - break; - } - else if (!hasRead) - { - hasAncestryRead = false; - break; - } - } - } - else - { - hasAncestryRead = containersToRoot(f).stream().allMatch(p -> { - if (!permission.containsKey(p.getId())) - permission.put(p.getId(), p.hasPermission(user, ReadPermission.class)); - return permission.get(p.getId()); - }); - } - - if (hasPolicyRead && hasAncestryRead && inActivePath) - { - // Is in the direct readable lineage of the current container - addFolder = true; - } - else if (hasPolicyRead && f.getParent().equals(c.getParent())) - { - // Is a readable sibling of the current container - addFolder = true; - } - else if (hasAncestryRead) - { - // Is a part of a fully readable ancestry - addFolder = true; - } - } - - if (!addFolder) - LOG.debug("isNavAccessOpen restriction: \"" + f.getPath() + "\""); - } - - if (addFolder) - { - containersInTree.add(f); - m.put(f.getId(), t); - } - } - - //Ensure parents of any accessible folder are in the tree. If not add them with no link. - for (Container treeContainer : containersInTree) - { - if (!treeContainer.equals(project) && !containersInTree.contains(treeContainer.getParent())) - { - Set containersToRoot = containersToRoot(treeContainer); - //Possible will be added more than once, if several children are accessible, but that's OK... - for (Container missing : containersToRoot) - { - if (!m.containsKey(missing.getId())) - { - if (isNavAccessOpen) - { - NavTree noLinkTree = new NavTree(missing.getName()); - noLinkTree.setId(missing.getId()); - m.put(missing.getId(), noLinkTree); - } - else - { - if (!permission.containsKey(missing.getId())) - permission.put(missing.getId(), missing.hasPermission(user, ReadPermission.class)); - - if (!permission.get(missing.getId())) - { - NavTree noLinkTree = new NavTree(missing.getName()); - m.put(missing.getId(), noLinkTree); - } - else - { - NavTree linkTree = new NavTree(missing.getName()); - ActionURL url = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(missing); - linkTree.setHref(url.getEncodedLocalURIString()); - m.put(missing.getId(), linkTree); - } - } - } - } - } - } - - for (Container f : folders) - { - if (f.getId().equals(projectId)) - continue; - - NavTree child = m.get(f.getId()); - if (null == child) - continue; - - NavTree parent = m.get(f.getParent().getId()); - assert null != parent; //This should not happen anymore, we assure all parents are in tree. - if (null != parent) - parent.addChild(child); - } - - NavTree projectTree = m.get(projectId); - - projectTree.setId(cacheKey); - - NavTreeManager.cacheTree(projectTree, user); - return projectTree; - } - finally - { - assert SecurityLogger.outdent(); - } - } - - public static Set containersToRoot(Container child) - { - Set containersOnPath = new HashSet<>(); - Container current = child; - while (current != null && !current.isRoot()) - { - containersOnPath.add(current); - current = current.getParent(); - } - - return containersOnPath; - } - - /** - * Provides a sorted list of containers from the root to the child container provided. - * It does not include the root node. - * @param child Container from which the search is sourced. - * @return List sorted in order of distance from root. - */ - public static List containersToRootList(Container child) - { - List containers = new ArrayList<>(); - Container current = child; - while (current != null && !current.isRoot()) - { - containers.add(current); - current = current.getParent(); - } - - Collections.reverse(containers); - return containers; - } - - // Move a container to another part of the container tree. Careful: this method DOES NOT prevent you from orphaning - // an entire tree (e.g., by setting a container's parent to one of its children); the UI in AdminController does this. - // - // NOTE: Beware side-effect of changing ACLs and GROUPS if a container changes projects - // - // @return true if project has changed (should probably redirect to security page) - public static boolean move(Container c, final Container newParent, User user) throws ValidationException - { - if (!isRenameable(c)) - { - throw new IllegalArgumentException("Can't move container " + c.getPath()); - } - - try (QuietCloser ignored = lockForMutation(MutatingOperation.move, c)) - { - List errors = new ArrayList<>(); - for (ContainerListener listener : getListeners()) - { - try - { - errors.addAll(listener.canMove(c, newParent, user)); - } - catch (Exception e) - { - ExceptionUtil.logExceptionToMothership(null, new IllegalStateException(listener.getClass().getName() + ".canMove() threw an exception or violated @NotNull contract")); - } - } - if (!errors.isEmpty()) - { - ValidationException exception = new ValidationException(); - for (String error : errors) - { - exception.addError(new SimpleValidationError(error)); - } - throw exception; - } - - if (c.getParent().getId().equals(newParent.getId())) - return false; - - Container oldParent = c.getParent(); - Container oldProject = c.getProject(); - Container newProject = newParent.isRoot() ? c : newParent.getProject(); - - boolean changedProjects = !oldProject.getId().equals(newProject.getId()); - - // Synchronize the transaction, but not the listeners -- see #9901 - try (DbScope.Transaction t = ensureTransaction()) - { - new SqlExecutor(CORE.getSchema()).execute("UPDATE " + CORE.getTableInfoContainers() + " SET Parent = ? WHERE EntityId = ?", newParent.getId(), c.getId()); - - // Refresh the container directly from the database so the container reflects the new parent, isProject(), etc. - c = getForRowId(c.getRowId()); - - // this could be done in the trigger, but I prefer to put it in the transaction - if (changedProjects) - SecurityManager.changeProject(c, oldProject, newProject, user); - - clearCache(); - - try - { - ExperimentService.get().moveContainer(c, oldParent, newParent); - } - catch (ExperimentException e) - { - throw new RuntimeException(e); - } - - // Clear after the commit has propagated the state to other threads and transactions - // Do this in a commit task in case we've joined another existing DbScope.Transaction instead of starting our own - t.addCommitTask(() -> - { - clearCache(); - getChildrenMap(newParent); // reload the cache - }, DbScope.CommitTaskOption.POSTCOMMIT); - - t.commit(); - } - - Container newContainer = getForId(c.getId()); - fireMoveContainer(newContainer, oldParent, user); - - return changedProjects; - } - } - - public static void rename(@NotNull Container c, User user, String name) - { - rename(c, user, name, c.getTitle(), false); - } - - /** - * Transacted method to rename a container. Optionally, supports updating the title and aliasing the - * original container path when the name is changed (as name changes result in a new container path). - */ - public static Container rename(@NotNull Container c, User user, String name, @Nullable String title, boolean addAlias) - { - try (QuietCloser ignored = lockForMutation(MutatingOperation.rename, c); - DbScope.Transaction tx = ensureTransaction()) - { - final String oldName = c.getName(); - final String newName = StringUtils.trimToNull(name); - boolean isRenaming = !oldName.equals(newName); - StringBuilder errors = new StringBuilder(); - - // Rename - if (isRenaming) - { - // Issue 16221: Don't allow renaming of system reserved folders (e.g. /Shared, home, root, etc). - if (!isRenameable(c)) - throw new ApiUsageException("This folder may not be renamed as it is reserved by the system."); - - if (!Container.isLegalName(newName, c.isProject(), errors)) - throw new ApiUsageException(errors.toString()); - - // Issue 19061: Unable to do case-only container rename - if (c.getParent().hasChild(newName) && !c.equals(c.getParent().getChild(newName))) - { - if (c.getParent().isRoot()) - throw new ApiUsageException("The server already has a project with this name."); - throw new ApiUsageException("The " + (c.getParent().isProject() ? "project " : "folder ") + c.getParent().getPath() + " already has a folder with this name."); - } - - new SqlExecutor(CORE.getSchema()).execute("UPDATE " + CORE.getTableInfoContainers() + " SET Name=? WHERE EntityId=?", newName, c.getId()); - clearCache(); // Clear the entire cache, since containers cache their full paths - // Get new version since name has changed. - Container renamedContainer = getForId(c.getId()); - fireRenameContainer(renamedContainer, user, oldName); - // Clear again after the commit has propagated the state to other threads and transactions - // Do this in a commit task in case we've joined another existing DbScope.Transaction instead of starting our own - tx.addCommitTask(ContainerManager::clearCache, DbScope.CommitTaskOption.POSTCOMMIT); - - // Alias - if (addAlias) - { - // Intentionally use original container rather than the already renamedContainer - List newAliases = new ArrayList<>(getAliasesForContainer(c)); - newAliases.add(c.getPath()); - saveAliasesForContainer(c, newAliases, user); - } - } - - // Title - if (!c.getTitle().equals(title)) - { - if (!Container.isLegalTitle(title, errors)) - throw new ApiUsageException(errors.toString()); - updateTitle(c, title, user); - } - - tx.commit(); - } - catch (ValidationException e) - { - throw new IllegalArgumentException(e); - } - - return getForId(c.getId()); - } - - public static void setChildOrderToAlphabetical(Container parent) - { - setChildOrder(parent.getChildren(), true); - } - - public static void setChildOrder(Container parent, List orderedChildren) throws ContainerException - { - for (Container child : orderedChildren) - { - if (child == null || child.getParent() == null || !child.getParent().equals(parent)) // #13481 - throw new ContainerException("Invalid parent container of " + (child == null ? "null child container" : child.getPath())); - } - setChildOrder(orderedChildren, false); - } - - private static void setChildOrder(List siblings, boolean resetToAlphabetical) - { - try (DbScope.Transaction t = ensureTransaction()) - { - for (int index = 0; index < siblings.size(); index++) - { - Container current = siblings.get(index); - new SqlExecutor(CORE.getSchema()).execute("UPDATE " + CORE.getTableInfoContainers() + " SET SortOrder = ? WHERE EntityId = ?", - resetToAlphabetical ? 0 : index, current.getId()); - } - // Clear after the commit has propagated the state to other threads and transactions - // Do this in a commit task in case we've joined another existing DbScope.Transaction instead of starting our own - t.addCommitTask(ContainerManager::clearCache, DbScope.CommitTaskOption.POSTCOMMIT); - - t.commit(); - } - } - - private enum MutatingOperation - { - delete, - rename, - move - } - - private static final Map mutatingContainers = Collections.synchronizedMap(new IntHashMap<>()); - - private static QuietCloser lockForMutation(MutatingOperation op, Container c) - { - return lockForMutation(op, Collections.singletonList(c)); - } - - private static QuietCloser lockForMutation(MutatingOperation op, Collection containers) - { - List ids = new ArrayList<>(containers.size()); - synchronized (mutatingContainers) - { - for (Container container : containers) - { - MutatingOperation currentOp = mutatingContainers.get(container.getRowId()); - if (currentOp != null) - { - throw new ApiUsageException("Cannot start a " + op + " operation on " + container.getPath() + ". It is currently undergoing a " + currentOp); - } - ids.add(container.getRowId()); - } - ids.forEach(id -> mutatingContainers.put(id, op)); - } - return () -> - { - synchronized (mutatingContainers) - { - ids.forEach(mutatingContainers::remove); - } - }; - } - - // Delete containers from the database - private static boolean delete(final Collection containers, User user, @Nullable String comment) - { - // Do this check before we bother with any synchronization - for (Container container : containers) - { - if (!isDeletable(container)) - { - throw new ApiUsageException("Cannot delete container: " + container.getPath()); - } - } - - try (QuietCloser ignored = lockForMutation(MutatingOperation.delete, containers)) - { - boolean deleted = true; - for (Container c : containers) - { - deleted = deleted && delete(c, user, comment); - } - return deleted; - } - } - - // Delete a container from the database - private static boolean delete(final Container c, User user, @Nullable String comment) - { - // Verify method isn't called inappropriately - if (mutatingContainers.get(c.getRowId()) != MutatingOperation.delete) - { - throw new IllegalStateException("Container not flagged as being deleted: " + c.getPath()); - } - - LOG.debug("Starting container delete for " + c.getContainerNoun(true) + " " + c.getPath()); - - // Tell the search indexer to drop work for the container that's about to be deleted - SearchService.get().purgeForContainer(c); - - DbScope.RetryFn tryDeleteContainer = (tx) -> - { - // Verify that no children exist - Selector sel = new TableSelector(CORE.getTableInfoContainers(), new SimpleFilter(FieldKey.fromParts("Parent"), c), null); - - if (sel.exists()) - { - _removeFromCache(c, true); - return false; - } - - if (c.shouldRemoveFromPortal()) - { - // Need to remove portal page, too; container name is page's pageId and in container's parent container - Portal.PortalPage page = Portal.getPortalPage(c.getParent(), c.getName()); - if (null != page) // Be safe - Portal.deletePage(page); - - // Tell parent - setContainerTabDeleted(c.getParent(), c.getName(), c.getParent().getFolderType().getName()); - } - - fireDeleteContainer(c, user); - - SqlExecutor sqlExecutor = new SqlExecutor(CORE.getSchema()); - sqlExecutor.execute("DELETE FROM " + CORE.getTableInfoContainerAliases() + " WHERE ContainerRowId=?", c.getRowId()); - sqlExecutor.execute("DELETE FROM " + CORE.getTableInfoContainers() + " WHERE EntityId=?", c.getId()); - // now that the container is actually gone, delete all ACLs (better to have an ACL w/o object than object w/o ACL) - SecurityPolicyManager.removeAll(c); - // and delete all container-based sequences - DbSequenceManager.deleteAll(c); - - ExperimentService experimentService = ExperimentService.get(); - if (experimentService != null) - experimentService.removeContainerDataTypeExclusions(c.getId()); - - // After we've committed the transaction, be sure that we remove this container from the cache - // See https://www.labkey.org/issues/home/Developer/issues/details.view?issueId=17015 - tx.addCommitTask(() -> - { - // Be sure that we've waited until any threads that might be populating the cache have finished - // before we guarantee that we've removed this now-deleted container - DATABASE_QUERY_LOCK.lock(); - try - { - _removeFromCache(c, true); - } - finally - { - DATABASE_QUERY_LOCK.unlock(); - } - }, DbScope.CommitTaskOption.POSTCOMMIT); - String auditComment = c.getContainerNoun(true) + " " + c.getPath() + " was deleted"; - if (comment != null) - auditComment = auditComment.concat(". " + comment); - addAuditEvent(user, c, auditComment); - return true; - }; - - boolean success = CORE.getSchema().getScope().executeWithRetry(tryDeleteContainer); - if (success) - { - LOG.debug("Completed container delete for " + c.getContainerNoun(true) + " " + c.getPath()); - } - else - { - LOG.warn("Failed to delete container: " + c.getPath()); - } - return success; - } - - /** - * Delete a single container. Primarily for use by tests. - */ - public static boolean delete(final Container c, User user) - { - return delete(List.of(c), user, null); - } - - public static boolean isDeletable(Container c) - { - return !isSystemContainer(c); - } - - public static boolean isRenameable(Container c) - { - return !isSystemContainer(c); - } - - /** System containers include the root container, /Home, and /Shared */ - public static boolean isSystemContainer(Container c) - { - return c.equals(getRoot()) || c.equals(getHomeContainer()) || c.equals(getSharedContainer()); - } - - /** Has the container already been deleted or is it in the process of being deleted? */ - public static boolean exists(@Nullable Container c) - { - return c != null && null != getForId(c.getEntityId()) && mutatingContainers.get(c.getRowId()) != MutatingOperation.delete; - } - - public static void deleteAll(Container root, User user, @Nullable String comment) throws UnauthorizedException - { - if (!hasTreePermission(root, user, DeletePermission.class)) - throw new UnauthorizedException("You don't have delete permissions to all folders"); - - LOG.debug("Starting container (and children) delete for " + root.getContainerNoun(true) + " " + root.getPath()); - Set depthFirst = getAllChildrenDepthFirst(root); - depthFirst.add(root); - - delete(depthFirst, user, comment); - - LOG.debug("Completed container (and children) delete for " + root.getContainerNoun(true) + " " + root.getPath()); - } - - public static void deleteAll(Container root, User user) throws UnauthorizedException - { - deleteAll(root, user, null); - } - - private static void addAuditEvent(User user, Container c, String comment) - { - if (user != null) - { - AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, c, comment); - AuditLogService.get().addEvent(user, event); - } - } - - private static Set getAllChildrenDepthFirst(Container c) - { - Set set = new LinkedHashSet<>(); - getAllChildrenDepthFirst(c, set); - return set; - } - - private static void getAllChildrenDepthFirst(Container c, Collection list) - { - for (Container child : c.getChildren()) - { - getAllChildrenDepthFirst(child, list); - list.add(child); - } - } - - private static Container _getFromCachePath(Path path) - { - return CACHE_PATH.get(path); - } - - private static Container _addToCache(Container c) - { - assert DATABASE_QUERY_LOCK.isHeldByCurrentThread() : "Any cache modifications must be synchronized at a " + - "higher level so that we ensure that the container to be inserted still exists and hasn't been deleted"; - CACHE_ENTITY_ID.put(c.getEntityId(), c); - CACHE_PATH.put(c.getParsedPath(), c); - return c; - } - - private static void _clearChildrenFromCache(Container c) - { - CACHE_CHILDREN.remove(c.getEntityId()); - navTreeManageUncache(c); - } - - /** @param hierarchyChange whether the shape of the container tree has changed */ - private static void _removeFromCache(Container c, boolean hierarchyChange) - { - CACHE_ENTITY_ID.remove(c.getEntityId()); - CACHE_PATH.remove(c.getParsedPath()); - - if (hierarchyChange) - { - // This is strictly keeping track of the parent/child relationships themselves so it only needs to be - // cleared when the tree changes - CACHE_CHILDREN.clear(); - } - - navTreeManageUncache(c); - } - - public static void clearCache() - { - CACHE_PATH.clear(); - CACHE_ENTITY_ID.clear(); - CACHE_CHILDREN.clear(); - - // UNDONE: NavTreeManager should register a ContainerListener - NavTreeManager.uncacheAll(); - } - - private static void navTreeManageUncache(Container c) - { - // UNDONE: NavTreeManager should register a ContainerListener - NavTreeManager.uncacheTree(PROJECT_LIST_ID); - NavTreeManager.uncacheTree(getRoot().getId()); - - Container project = c.getProject(); - if (project != null) - { - NavTreeManager.uncacheTree(project.getId()); - NavTreeManager.uncacheTree(PROJECT_LIST_ID + project.getId()); - } - } - - public static void notifyContainerChange(String id, Property prop) - { - notifyContainerChange(id, prop, null); - } - - public static void notifyContainerChange(String id, Property prop, @Nullable User u) - { - if (_constructing.contains(new GUID(id))) - return; - - Container c = getForId(id); - if (null != c) - { - _removeFromCache(c, false); - c = getForId(id); // load a fresh container since the original might be stale. - if (null != c) - { - ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(c, u, prop, null, null); - firePropertyChangeEvent(evt); - } - } - } - - - /** Recursive, including root node */ - public static Set getAllChildren(Container root) - { - Set children = getAllChildrenDepthFirst(root); - children.add(root); - - return Collections.unmodifiableSet(children); - } - - /** - * Return all children of the root node, including root node, which have the given active module - */ - @NotNull - public static Set getAllChildrenWithModule(@NotNull Container root, @NotNull Module module) - { - Set children = new HashSet<>(); - for (Container candidate : getAllChildren(root)) - { - if (candidate.getActiveModules().contains(module)) - children.add(candidate); - } - return Collections.unmodifiableSet(children); - } - - public static long getContainerCount() - { - return new TableSelector(CORE.getTableInfoContainers()).getRowCount(); - } - - public static long getWorkbookCount() - { - return new TableSelector(CORE.getTableInfoContainers(), new SimpleFilter(FieldKey.fromParts("type"), "workbook"), null).getRowCount(); - } - - public static long getArchivedContainerCount() - { - return new TableSelector(CORE.getTableInfoContainers(), new SimpleFilter(FieldKey.fromParts("lockstate"), "Archived"), null).getRowCount(); - } - - public static long getAuditCommentRequiredCount() - { - SQLFragment sql = new SQLFragment( - "SELECT COUNT(*) FROM\n" + - " core.containers c\n" + - " JOIN prop.propertysets ps on c.entityid = ps.objectid\n" + - " JOIN prop.properties p on p.\"set\" = ps.\"set\"\n" + - "WHERE ps.category = '" + AUDIT_SETTINGS_PROPERTY_SET_NAME + "' AND p.name='"+ REQUIRE_USER_COMMENTS_PROPERTY_NAME + "' and p.value='true'"); - return new SqlSelector(CORE.getSchema(), sql).getObject(Long.class); - } - - - /** Retrieve entire container hierarchy */ - public static MultiValuedMap getContainerTree() - { - final MultiValuedMap mm = new ArrayListValuedHashMap<>(); - - // Get all containers and parents - SqlSelector selector = new SqlSelector(CORE.getSchema(), "SELECT Parent, EntityId FROM " + CORE.getTableInfoContainers() + " ORDER BY SortOrder, LOWER(Name) ASC"); - - selector.forEach(rs -> { - String parentId = rs.getString(1); - Container parent = (parentId != null ? getForId(parentId) : null); - Container child = getForId(rs.getString(2)); - - if (null != child) - mm.put(parent, child); - }); - - return mm; - } - - /** - * Returns a branch of the container tree including only the root and its descendants - * @param root The root container - * @return MultiMap of containers including root and its descendants - */ - public static MultiValuedMap getContainerTree(Container root) - { - //build a multimap of only the container ids - final MultiValuedMap mmIds = new ArrayListValuedHashMap<>(); - - // Get all containers and parents - Selector selector = new SqlSelector(CORE.getSchema(), "SELECT Parent, EntityId FROM " + CORE.getTableInfoContainers() + " ORDER BY SortOrder, LOWER(Name) ASC"); - - selector.forEach(rs -> mmIds.put(rs.getString(1), rs.getString(2))); - - //now find the root and build a MultiMap of it and its descendants - MultiValuedMap mm = new ArrayListValuedHashMap<>(); - mm.put(null, root); - addChildren(root, mmIds, mm); - return mm; - } - - private static void addChildren(Container c, MultiValuedMap mmIds, MultiValuedMap mm) - { - Collection childIds = mmIds.get(c.getId()); - if (null != childIds) - { - for (String childId : childIds) - { - Container child = getForId(childId); - if (null != child) - { - mm.put(c, child); - addChildren(child, mmIds, mm); - } - } - } - } - - public static Set getContainerSet(MultiValuedMap mm, User user, Class perm) - { - Collection containers = mm.values(); - if (null == containers) - return new HashSet<>(); - - return containers - .stream() - .filter(c -> c.hasPermission(user, perm)) - .collect(Collectors.toSet()); - } - - - public static SQLFragment getIdsAsCsvList(Set containers, SqlDialect d) - { - if (containers.isEmpty()) - return new SQLFragment("(NULL)"); // WHERE x IN (NULL) should match no rows - - SQLFragment csvList = new SQLFragment("("); - String comma = ""; - for (Container container : containers) - { - csvList.append(comma); - comma = ","; - csvList.appendValue(container, d); - } - csvList.append(")"); - - return csvList; - } - - - public static List getIds(User user, Class perm) - { - Set containers = getContainerSet(getContainerTree(), user, perm); - - List ids = new ArrayList<>(containers.size()); - - for (Container c : containers) - ids.add(c.getId()); - - return ids; - } - - - // - // ContainerListener - // - - public interface ContainerListener extends PropertyChangeListener - { - enum Order {First, Last} - - /** Called after a new container has been created */ - void containerCreated(Container c, User user); - - default void containerCreated(Container c, User user, @Nullable String auditMsg) - { - containerCreated(c, user); - } - - /** Called immediately prior to deleting the row from core.containers */ - void containerDeleted(Container c, User user); - - /** Called after the container has been moved to its new parent */ - void containerMoved(Container c, Container oldParent, User user); - - /** - * Called prior to moving a container, to find out if there are any issues that would prevent a successful move - * @return a list of errors that should prevent the move from happening, if any - */ - @NotNull - Collection canMove(Container c, Container newParent, User user); - - @Override - void propertyChange(PropertyChangeEvent evt); - } - - public static abstract class AbstractContainerListener implements ContainerListener - { - @Override - public void containerCreated(Container c, User user) - {} - - @Override - public void containerDeleted(Container c, User user) - {} - - @Override - public void containerMoved(Container c, Container oldParent, User user) - {} - - @NotNull - @Override - public Collection canMove(Container c, Container newParent, User user) - { - return Collections.emptyList(); - } - - @Override - public void propertyChange(PropertyChangeEvent evt) - {} - } - - - public static class ContainerPropertyChangeEvent extends PropertyChangeEvent implements PropertyChange - { - public final Property property; - public final Container container; - public User user; - - public ContainerPropertyChangeEvent(Container c, @Nullable User user, Property p, Object oldValue, Object newValue) - { - super(c, p.name(), oldValue, newValue); - container = c; - this.user = user; - property = p; - } - - public ContainerPropertyChangeEvent(Container c, Property p, Object oldValue, Object newValue) - { - this(c, null, p, oldValue, newValue); - } - - @Override - public Property getProperty() - { - return property; - } - } - - - // Thread-safe list implementation that allows iteration and modifications without external synchronization - private static final List _listeners = new CopyOnWriteArrayList<>(); - private static final List _laterListeners = new CopyOnWriteArrayList<>(); - - // These listeners are executed in the order they are registered, before the "Last" listeners - public static void addContainerListener(ContainerListener listener) - { - addContainerListener(listener, ContainerListener.Order.First); - } - - - // Explicitly request "Last" ordering via this method. "Last" listeners execute after all "First" listeners. - public static void addContainerListener(ContainerListener listener, ContainerListener.Order order) - { - if (ContainerListener.Order.First == order) - _listeners.add(listener); - else - _laterListeners.add(listener); - } - - - public static void removeContainerListener(ContainerListener listener) - { - _listeners.remove(listener); - _laterListeners.remove(listener); - } - - - private static List getListeners() - { - List combined = new ArrayList<>(_listeners.size() + _laterListeners.size()); - combined.addAll(_listeners); - combined.addAll(_laterListeners); - - return combined; - } - - - private static List getListenersReversed() - { - List combined = new LinkedList<>(); - - // Copy to guarantee consistency between .listIterator() and .size() - List copy = new ArrayList<>(_listeners); - ListIterator iter = copy.listIterator(copy.size()); - - // Iterate in reverse - while(iter.hasPrevious()) - combined.add(iter.previous()); - - // Copy to guarantee consistency between .listIterator() and .size() - // Add elements from the laterList in reverse order so that Core is fired last - List laterCopy = new ArrayList<>(_laterListeners); - ListIterator laterIter = laterCopy.listIterator(laterCopy.size()); - - // Iterate in reverse - while(laterIter.hasPrevious()) - combined.add(laterIter.previous()); - - return combined; - } - - - protected static void fireCreateContainer(Container c, User user, @Nullable String auditMsg) - { - List list = getListeners(); - - for (ContainerListener cl : list) - { - try - { - cl.containerCreated(c, user, auditMsg); - } - catch (Throwable t) - { - LOG.error("fireCreateContainer for " + cl.getClass().getName(), t); - } - } - } - - - protected static void fireDeleteContainer(Container c, User user) - { - List list = getListenersReversed(); - - for (ContainerListener l : list) - { - LOG.debug("Deleting " + c.getPath() + ": fireDeleteContainer for " + l.getClass().getName()); - try - { - l.containerDeleted(c, user); - } - catch (RuntimeException e) - { - LOG.error("fireDeleteContainer for " + l.getClass().getName(), e); - - // Fail fast (first Throwable aborts iteration), #17560 - throw e; - } - } - } - - - protected static void fireRenameContainer(Container c, User user, String oldValue) - { - ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(c, user, Property.Name, oldValue, c.getName()); - firePropertyChangeEvent(evt); - } - - - protected static void fireMoveContainer(Container c, Container oldParent, User user) - { - List list = getListeners(); - - for (ContainerListener cl : list) - { - // While we would ideally transact the full container move, that will likely cause long-blocking - // queries and/or deadlocks. For now, at least transact each separate move handler independently - try (DbScope.Transaction transaction = CoreSchema.getInstance().getSchema().getScope().ensureTransaction()) - { - cl.containerMoved(c, oldParent, user); - transaction.commit(); - } - } - ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(c, user, Property.Parent, oldParent, c.getParent()); - firePropertyChangeEvent(evt); - } - - - public static void firePropertyChangeEvent(ContainerPropertyChangeEvent evt) - { - if (_constructing.contains(evt.container.getEntityId())) - return; - - List list = getListeners(); - for (ContainerListener l : list) - { - try - { - l.propertyChange(evt); - } - catch (Throwable t) - { - LOG.error("firePropertyChangeEvent for " + l.getClass().getName(), t); - } - } - } - - private static final List MODULE_DEPENDENCY_PROVIDERS = new CopyOnWriteArrayList<>(); - - public static void registerModuleDependencyProvider(ModuleDependencyProvider provider) - { - MODULE_DEPENDENCY_PROVIDERS.add(provider); - } - - public static void forEachModuleDependencyProvider(Consumer action) - { - MODULE_DEPENDENCY_PROVIDERS.forEach(action); - } - - // Compliance module adds a locked project handler that checks permissions; without that, this implementation - // is used, and projects are never locked - static volatile LockedProjectHandler LOCKED_PROJECT_HANDLER = (project, user, contextualRoles, lockState) -> false; - - // Replaces any previously set LockedProjectHandler - public static void setLockedProjectHandler(LockedProjectHandler handler) - { - LOCKED_PROJECT_HANDLER = handler; - } - - public static Container createDefaultSupportContainer() - { - LOG.info("Creating default support container: " + DEFAULT_SUPPORT_PROJECT_PATH); - // create a "support" container. Admins can do anything, - // Users can read/write, Guests can read. - return bootstrapContainer(DEFAULT_SUPPORT_PROJECT_PATH, - RoleManager.getRole(AuthorRole.class), - RoleManager.getRole(ReaderRole.class) - ); - } - - public static void removeDefaultSupportContainer(User user) - { - Container support = getDefaultSupportContainer(); - if (support != null) - { - LOG.info("Removing default support container: " + DEFAULT_SUPPORT_PROJECT_PATH); - ContainerManager.delete(support, user); - } - } - - public static Container getDefaultSupportContainer() - { - return getForPath(DEFAULT_SUPPORT_PROJECT_PATH); - } - - public static List getAliasesForContainer(Container c) - { - return Collections.unmodifiableList(new SqlSelector(CORE.getSchema(), - new SQLFragment("SELECT Path FROM " + CORE.getTableInfoContainerAliases() + " WHERE ContainerRowId = ? ORDER BY Path", - c.getRowId())).getArrayList(String.class)); - } - - @Nullable - public static Container resolveContainerPathAlias(String path) - { - return resolveContainerPathAlias(path, false); - } - - @Nullable - private static Container resolveContainerPathAlias(String path, boolean top) - { - // Strip any trailing slashes - while (path.endsWith("/")) - { - path = path.substring(0, path.length() - 1); - } - - // Simple case -- resolve directly (sans alias) - Container aliased = getForPath(path); - if (aliased != null) - return aliased; - - // Simple case -- directly resolve from database - aliased = getForPathAlias(path); - if (aliased != null) - return aliased; - - // At the leaf and the container was not found - if (top) - return null; - - List splits = Arrays.asList(path.split("/")); - String subPath = ""; - for (int i=0; i < splits.size()-1; i++) // minus 1 due to leaving off last container - { - if (!splits.get(i).isEmpty()) - subPath += "/" + splits.get(i); - } - - aliased = resolveContainerPathAlias(subPath, false); - - if (aliased == null) - return null; - - String leafPath = aliased.getPath() + "/" + splits.get(splits.size()-1); - return resolveContainerPathAlias(leafPath, true); - } - - @Nullable - private static Container getForPathAlias(String path) - { - // We store the path as lower-case, so we don't need to also LOWER() on the value in core.ContainerAliases, letting the DB use the index - Container[] ret = new SqlSelector(CORE.getSchema(), - "SELECT * FROM " + CORE.getTableInfoContainers() + " c, " + CORE.getTableInfoContainerAliases() + " ca WHERE ca.ContainerRowId = c.RowId AND ca.path = LOWER(?)", - path).getArray(Container.class); - - return ret.length == 0 ? null : ret[0]; - } - - public static Container getMoveTargetContainer(@Nullable String queryName, @NotNull Container sourceContainer, User user, @Nullable String targetIdOrPath, Errors errors) - { - if (targetIdOrPath == null) - { - errors.reject(ERROR_GENERIC, "A target container must be specified for the move operation."); - return null; - } - - Container _targetContainer = getContainerForIdOrPath(targetIdOrPath); - if (_targetContainer == null) - { - errors.reject(ERROR_GENERIC, "The target container was not found: " + targetIdOrPath + "."); - return null; - } - - if (!_targetContainer.hasPermission(user, InsertPermission.class)) - { - String _queryName = queryName == null ? "this table" : "'" + queryName + "'"; - errors.reject(ERROR_GENERIC, "You do not have permission to move rows from " + _queryName + " to the target container: " + targetIdOrPath + "."); - return null; - } - - if (!isValidTargetContainer(sourceContainer, _targetContainer)) - { - errors.reject(ERROR_GENERIC, "Invalid target container for the move operation: " + targetIdOrPath + "."); - return null; - } - return _targetContainer; - } - - private static Container getContainerForIdOrPath(String targetContainer) - { - Container c = ContainerManager.getForId(targetContainer); - if (c == null) - c = ContainerManager.getForPath(targetContainer); - - return c; - } - - // targetContainer must be in the same app project at this time - // i.e. child of current project, project of current child, sibling within project - private static boolean isValidTargetContainer(Container current, Container target) - { - if (current.isRoot() || target.isRoot()) - return false; - - // Allow moving to the current container since we now allow the chosen entities to be from different containers - if (current.equals(target)) - return true; - - boolean moveFromProjectToChild = current.isProject() && target.getParent().equals(current); - boolean moveFromChildToProject = !current.isProject() && current.getParent().isProject() && current.getParent().equals(target); - boolean moveFromChildToSibling = !current.isProject() && current.getParent().isProject() && current.getParent().equals(target.getParent()); - - return moveFromProjectToChild || moveFromChildToProject || moveFromChildToSibling; - } - - /** - * Updates the container of specified rows in the provided database table. Optionally, the modification timestamp - * and the user who made the modification can also be updated if specified. - * - * @param dataTable The table where the container update should be applied. - * @param idField The name of the identifier field used to locate the rows to update. - * @param ids A collection of identifier values specifying the rows to be updated. - * @param targetContainer The target container to set for the specified rows. - * @param user The user performing the update. If null, modified/modifiedBy details are not updated. - * @param withModified If true, updates the modified timestamp and the user who made the modification. - * @return The number of rows updated in the table. - */ - public static int updateContainer(TableInfo dataTable, String idField, Collection ids, Container targetContainer, @Nullable User user, boolean withModified) - { - try (DbScope.Transaction transaction = dataTable.getSchema().getScope().ensureTransaction()) - { - SQLFragment dataUpdate = new SQLFragment("UPDATE ").append(dataTable) - .append(" SET container = ").appendValue(targetContainer); - if (withModified) - { - assert user != null : "User must be specified when updating modified/modifiedBy details."; - dataUpdate.append(", modified = ").appendValue(new Date()); - dataUpdate.append(", modifiedby = ").appendValue(user.getUserId()); - } - dataUpdate.append(" WHERE ").appendIdentifier(idField); - dataTable.getSchema().getSqlDialect().appendInClauseSql(dataUpdate, ids); - int numUpdated = new SqlExecutor(dataTable.getSchema()).execute(dataUpdate); - transaction.commit(); - - return numUpdated; - } - } - - /** - * If a container at the given path does not exist, create one and set permissions. If the container does exist, - * permissions are only set if there is no explicit ACL for the container. This prevents us from resetting - * permissions if all users are dropped. Implicitly done as an admin-level service user. - */ - @NotNull - public static Container bootstrapContainer(String path, @NotNull Role userRole, @Nullable Role guestRole) - { - Container c = null; - User user = User.getAdminServiceUser(); - - try - { - c = getForPath(path); - } - catch (RootContainerException e) - { - // Ignore this -- root doesn't exist yet - } - boolean newContainer = false; - - if (c == null) - { - LOG.debug("Creating new container for path '" + path + "'"); - newContainer = true; - c = ensureContainer(path, user); - } - - // Only set permissions if there are no explicit permissions - // set for this object or we just created it - Integer policyCount = null; - if (!newContainer) - { - policyCount = new SqlSelector(CORE.getSchema(), - "SELECT COUNT(*) FROM " + CORE.getTableInfoPolicies() + " WHERE ResourceId = ?", - c.getId()).getObject(Integer.class); - } - - if (newContainer || 0 == policyCount.intValue()) - { - LOG.debug("Setting permissions for '" + path + "'"); - MutableSecurityPolicy policy = new MutableSecurityPolicy(c); - policy.addRoleAssignment(SecurityManager.getGroup(Group.groupUsers), userRole); - if (guestRole != null) - policy.addRoleAssignment(SecurityManager.getGroup(Group.groupGuests), guestRole); - SecurityPolicyManager.savePolicy(policy, user); - } - - return c; - } - - /** - * @param container the container being created. May be null if we haven't actually created it yet - * @param parent the parent of the container being created. Used in case the container doesn't actually exist yet. - * @return the list of standard steps and any extra ones based on the container's FolderType - */ - public static List getCreateContainerWizardSteps(@Nullable Container container, @NotNull Container parent) - { - List navTrail = new ArrayList<>(); - - boolean isProject = parent.isRoot(); - - navTrail.add(new NavTree(isProject ? "Create Project" : "Create Folder")); - navTrail.add(new NavTree("Users / Permissions")); - if (isProject) - navTrail.add(new NavTree("Project Settings")); - if (container != null) - navTrail.addAll(container.getFolderType().getExtraSetupSteps(container)); - return navTrail; - } - - @TestTimeout(120) @TestWhen(TestWhen.When.BVT) - public static class TestCase extends Assert implements ContainerListener - { - Map _containers = new HashMap<>(); - Container _testRoot = null; - - @Before - public void setUp() - { - if (null == _testRoot) - { - Container junit = JunitUtil.getTestContainer(); - _testRoot = ensureContainer(junit, "ContainerManager$TestCase-" + GUID.makeGUID(), TestContext.get().getUser()); - addContainerListener(this); - } - } - - @After - public void tearDown() - { - removeContainerListener(this); - if (null != _testRoot) - deleteAll(_testRoot, TestContext.get().getUser()); - } - - @Test - public void testImproperFolderNamesBlocked() - { - String[] badNames = {"", "f\\o", "f/o", "f\\\\o", "foo;", "@foo", "foo" + '\u001F', '\u0000' + "foo", "fo" + '\u007F' + "o", "" + '\u009F'}; - - for (String name: badNames) - { - try - { - Container c = createContainer(_testRoot, name, TestContext.get().getUser()); - try - { - assertTrue(delete(c, TestContext.get().getUser())); - } - catch (Exception ignored) {} - fail("Should have thrown exception when trying to create container with name: " + name); - } - catch (ApiUsageException e) - { - // Do nothing, this is expected - } - } - } - - @Test - public void testCreateDeleteContainers() - { - int count = 20; - Random random = new Random(); - MultiValuedMap mm = new ArrayListValuedHashMap<>(); - - for (int i = 1; i <= count; i++) - { - int parentId = random.nextInt(i); - String parentName = 0 == parentId ? _testRoot.getName() : String.valueOf(parentId); - String childName = String.valueOf(i); - mm.put(parentName, childName); - } - - logNode(mm, _testRoot.getName(), 0); - for (int i=0; i<2; i++) //do this twice to make sure the containers were *really* deleted - { - createContainers(mm, _testRoot.getName(), _testRoot); - assertEquals(count, _containers.size()); - cleanUpChildren(mm, _testRoot.getName(), _testRoot); - assertEquals(0, _containers.size()); - } - } - - @Test - public void testCache() - { - assertEquals(0, _containers.size()); - assertEquals(0, getChildren(_testRoot).size()); - - Container one = createContainer(_testRoot, "one", TestContext.get().getUser()); - assertEquals(1, _containers.size()); - assertEquals(1, getChildren(_testRoot).size()); - assertEquals(0, getChildren(one).size()); - - Container oneA = createContainer(one, "A", TestContext.get().getUser()); - assertEquals(2, _containers.size()); - assertEquals(1, getChildren(_testRoot).size()); - assertEquals(1, getChildren(one).size()); - assertEquals(0, getChildren(oneA).size()); - - Container oneB = createContainer(one, "B", TestContext.get().getUser()); - assertEquals(3, _containers.size()); - assertEquals(1, getChildren(_testRoot).size()); - assertEquals(2, getChildren(one).size()); - assertEquals(0, getChildren(oneB).size()); - - Container deleteme = createContainer(one, "deleteme", TestContext.get().getUser()); - assertEquals(4, _containers.size()); - assertEquals(1, getChildren(_testRoot).size()); - assertEquals(3, getChildren(one).size()); - assertEquals(0, getChildren(deleteme).size()); - - assertTrue(delete(deleteme, TestContext.get().getUser())); - assertEquals(3, _containers.size()); - assertEquals(1, getChildren(_testRoot).size()); - assertEquals(2, getChildren(one).size()); - - Container oneC = createContainer(one, "C", TestContext.get().getUser()); - assertEquals(4, _containers.size()); - assertEquals(1, getChildren(_testRoot).size()); - assertEquals(3, getChildren(one).size()); - assertEquals(0, getChildren(oneC).size()); - - assertTrue(delete(oneC, TestContext.get().getUser())); - assertTrue(delete(oneB, TestContext.get().getUser())); - assertEquals(1, getChildren(one).size()); - - assertTrue(delete(oneA, TestContext.get().getUser())); - assertEquals(0, getChildren(one).size()); - - assertTrue(delete(one, TestContext.get().getUser())); - assertEquals(0, getChildren(_testRoot).size()); - assertEquals(0, _containers.size()); - } - - @Test - public void testFolderType() - { - // Test all folder types - List folderTypes = new ArrayList<>(FolderTypeManager.get().getAllFolderTypes()); - for (FolderType folderType : folderTypes) - { - if (!folderType.isProjectOnlyType()) // Dataspace can't be subfolder - testOneFolderType(folderType); - } - } - - private void testOneFolderType(FolderType folderType) - { - LOG.info("testOneFolderType(" + folderType.getName() + "): creating container"); - Container newFolder = createContainer(_testRoot, "folderTypeTest", TestContext.get().getUser()); - FolderType ft = newFolder.getFolderType(); - assertEquals(FolderType.NONE, ft); - - Container newFolderFromCache = getForId(newFolder.getId()); - assertNotNull(newFolderFromCache); - assertEquals(FolderType.NONE, newFolderFromCache.getFolderType()); - LOG.info("testOneFolderType(" + folderType.getName() + "): setting folder type"); - newFolder.setFolderType(folderType, TestContext.get().getUser()); - - newFolderFromCache = getForId(newFolder.getId()); - assertNotNull(newFolderFromCache); - assertEquals(newFolderFromCache.getFolderType().getName(), folderType.getName()); - assertEquals(newFolderFromCache.getFolderType().getDescription(), folderType.getDescription()); - - LOG.info("testOneFolderType(" + folderType.getName() + "): deleteAll"); - deleteAll(newFolder, TestContext.get().getUser()); // There might be subfolders because of container tabs - LOG.info("testOneFolderType(" + folderType.getName() + "): deleteAll complete"); - Container deletedContainer = getForId(newFolder.getId()); - - if (deletedContainer != null) - { - fail("Expected container with Id " + newFolder.getId() + " to be deleted, but found " + deletedContainer + ". Folder type was " + folderType); - } - } - - private static void createContainers(MultiValuedMap mm, String name, Container parent) - { - Collection nodes = mm.get(name); - - if (null == nodes) - return; - - for (String childName : nodes) - { - Container child = createContainer(parent, childName, TestContext.get().getUser()); - createContainers(mm, childName, child); - } - } - - private static void cleanUpChildren(MultiValuedMap mm, String name, Container parent) - { - Collection nodes = mm.get(name); - - if (null == nodes) - return; - - for (String childName : nodes) - { - Container child = getForPath(makePath(parent, childName)); - cleanUpChildren(mm, childName, child); - assertTrue(delete(child, TestContext.get().getUser())); - } - } - - private static void logNode(MultiValuedMap mm, String name, int offset) - { - Collection nodes = mm.get(name); - - if (null == nodes) - return; - - for (String childName : nodes) - { - LOG.debug(StringUtils.repeat(" ", offset) + childName); - logNode(mm, childName, offset + 1); - } - } - - // ContainerListener - @Override - public void propertyChange(PropertyChangeEvent evt) - { - } - - @Override - public void containerCreated(Container c, User user) - { - if (null == _testRoot || !c.getParsedPath().startsWith(_testRoot.getParsedPath())) - return; - _containers.put(c.getParsedPath(), c); - } - - - @Override - public void containerDeleted(Container c, User user) - { - _containers.remove(c.getParsedPath()); - } - - @Override - public void containerMoved(Container c, Container oldParent, User user) - { - } - - @NotNull - @Override - public Collection canMove(Container c, Container newParent, User user) - { - return Collections.emptyList(); - } - } - - static - { - ObjectFactory.Registry.register(Container.class, new ContainerFactory()); - } - - public static class ContainerFactory implements ObjectFactory - { - @Override - public Container fromMap(Map m) - { - throw new UnsupportedOperationException(); - } - - @Override - public Container fromMap(Container bean, Map m) - { - throw new UnsupportedOperationException(); - } - - @Override - public Map toMap(Container bean, Map m) - { - throw new UnsupportedOperationException(); - } - - @Override - public Container handle(ResultSet rs) throws SQLException - { - String id; - Container d; - String parentId = rs.getString("Parent"); - String name = rs.getString("Name"); - id = rs.getString("EntityId"); - int rowId = rs.getInt("RowId"); - int sortOrder = rs.getInt("SortOrder"); - Date created = rs.getTimestamp("Created"); - int createdBy = rs.getInt("CreatedBy"); - // _ts - String description = rs.getString("Description"); - String type = rs.getString("Type"); - String title = rs.getString("Title"); - boolean searchable = rs.getBoolean("Searchable"); - String lockStateString = rs.getString("LockState"); - LockState lockState = null != lockStateString ? Enums.getIfPresent(LockState.class, lockStateString).or(LockState.Unlocked) : LockState.Unlocked; - - LocalDate expirationDate = rs.getObject("ExpirationDate", LocalDate.class); - - // Could be running upgrade code before these recent columns have been added to the table. Use a find map - // to determine if they are present. Issue 51692. These checks could be removed after creation of these - // columns is incorporated into the bootstrap scripts. - Map findMap = ResultSetUtil.getFindMap(rs.getMetaData()); - Long fileRootSize = findMap.containsKey("FileRootSize") ? (Long)rs.getObject("FileRootSize") : null; // getObject() and cast because getLong() returns 0 for null - LocalDateTime fileRootLastCrawled = findMap.containsKey("FileRootLastCrawled") ? rs.getObject("FileRootLastCrawled", LocalDateTime.class) : null; - - Container dirParent = null; - if (null != parentId) - dirParent = getForId(parentId); - - d = new Container(dirParent, name, id, rowId, sortOrder, created, createdBy, searchable); - d.setDescription(description); - d.setType(type); - d.setTitle(title); - d.setLockState(lockState); - d.setExpirationDate(expirationDate); - d.setFileRootSize(fileRootSize); - d.setFileRootLastCrawled(fileRootLastCrawled); - return d; - } - - @Override - public ArrayList handleArrayList(ResultSet rs) throws SQLException - { - ArrayList list = new ArrayList<>(); - while (rs.next()) - { - list.add(handle(rs)); - } - return list; - } - } - - public static Container createFakeContainer(@Nullable String name, @Nullable Container parent) - { - return new Container(parent, name, GUID.makeGUID(), 1, 0, new Date(), 0, false); - } -} +/* + * Copyright (c) 2005-2018 Fred Hutchinson Cancer Research Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.data; + +import com.google.common.base.Enums; +import org.apache.commons.collections4.MultiValuedMap; +import org.apache.commons.collections4.multimap.ArrayListValuedHashMap; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.xmlbeans.XmlObject; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.labkey.api.Constants; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.admin.FolderExportContext; +import org.labkey.api.admin.FolderImportContext; +import org.labkey.api.admin.FolderImporterImpl; +import org.labkey.api.admin.FolderWriterImpl; +import org.labkey.api.admin.StaticLoggerGetter; +import org.labkey.api.attachments.AttachmentParent; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.provider.ContainerAuditProvider; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.ConcurrentHashSet; +import org.labkey.api.collections.IntHashMap; +import org.labkey.api.data.Container.ContainerException; +import org.labkey.api.data.Container.LockState; +import org.labkey.api.data.PropertyManager.WritablePropertyMap; +import org.labkey.api.data.SimpleFilter.InClause; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.data.validator.ColumnValidators; +import org.labkey.api.event.PropertyChange; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.module.FolderType; +import org.labkey.api.module.FolderTypeManager; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.portal.ProjectUrls; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.SimpleValidationError; +import org.labkey.api.query.ValidationException; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.Group; +import org.labkey.api.security.MutableSecurityPolicy; +import org.labkey.api.security.SecurityLogger; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.security.SecurityPolicyManager; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.CreateProjectPermission; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.InsertPermission; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.security.roles.AuthorRole; +import org.labkey.api.security.roles.ReaderRole; +import org.labkey.api.security.roles.Role; +import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.settings.AppProps; +import org.labkey.api.test.TestTimeout; +import org.labkey.api.test.TestWhen; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JunitUtil; +import org.labkey.api.util.MinorConfigurationException; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Path; +import org.labkey.api.util.QuietCloser; +import org.labkey.api.util.ReentrantLockWithName; +import org.labkey.api.util.ResultSetUtil; +import org.labkey.api.util.TestContext; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.FolderTab; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.NavTreeManager; +import org.labkey.api.view.Portal; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.view.ViewContext; +import org.labkey.api.writer.MemoryVirtualFile; +import org.labkey.folder.xml.FolderDocument; +import org.labkey.remoteapi.collections.CaseInsensitiveHashMap; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; + +import java.beans.PropertyChangeEvent; +import java.beans.PropertyChangeListener; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static org.labkey.api.action.SpringActionController.ERROR_GENERIC; + +/** + * This class manages a hierarchy of collections, backed by a database table called Containers. + * Containers are named using filesystem-like paths e.g., /proteomics/comet/. Each path + * maps to a UID and set of permissions. The current security scheme allows ACLs + * to be specified explicitly on the directory or completely inherited. ACLs are not combined. + *

+ * NOTE: we act like java.io.File(). Paths start with forward-slash, but do not end with forward-slash. + * The root container's name is '/'. This means that it is not always the case that + * me.getPath() == me.getParent().getPath() + "/" + me.getName() + *

+ * The synchronization goals are to keep invalid containers from creeping into the cache. For example, once + * a container is deleted, it should never get put back in the cache. We accomplish this by synchronizing on + * the removal from the cache, and the database lookup/cache insertion. While a container is in the middle + * of being deleted, it's OK for other clients to see it because FKs enforce that it's always internally + * consistent, even if some data has already been deleted. + */ +public class ContainerManager +{ + private static final Logger LOG = LogHelper.getLogger(ContainerManager.class, "Container (projects, folders, and workbooks) retrieval and management"); + private static final CoreSchema CORE = CoreSchema.getInstance(); + + private static final String PROJECT_LIST_ID = "Projects"; + + public static final String HOME_PROJECT_PATH = "/home"; + public static final String DEFAULT_SUPPORT_PROJECT_PATH = HOME_PROJECT_PATH + "/support"; + + private static final Cache CACHE_PATH = CacheManager.getCache(Constants.getMaxContainers(), CacheManager.DAY, "Containers by Path"); + private static final Cache CACHE_ENTITY_ID = CacheManager.getCache(Constants.getMaxContainers(), CacheManager.DAY, "Containers by EntityId"); + private static final Cache> CACHE_CHILDREN = CacheManager.getCache(Constants.getMaxContainers(), CacheManager.DAY, "Child EntityIds of Containers"); + private static final ReentrantLock DATABASE_QUERY_LOCK = new ReentrantLockWithName(ContainerManager.class, "DATABASE_QUERY_LOCK"); + public static final String FOLDER_TYPE_PROPERTY_SET_NAME = "folderType"; + public static final String FOLDER_TYPE_PROPERTY_NAME = "name"; + public static final String FOLDER_TYPE_PROPERTY_TABTYPE_OVERRIDDEN = "ctFolderTypeOverridden"; + public static final String TABFOLDER_CHILDREN_DELETED = "tabChildrenDeleted"; + public static final String AUDIT_SETTINGS_PROPERTY_SET_NAME = "containerAuditSettings"; + public static final String REQUIRE_USER_COMMENTS_PROPERTY_NAME = "requireUserComments"; + + private static final List _resourceProviders = new CopyOnWriteArrayList<>(); + + // containers that are being constructed, used to suppress events before fireCreateContainer() + private static final Set _constructing = new ConcurrentHashSet<>(); + + + /** enum of properties you can see in property change events */ + public enum Property + { + Name, + Parent, + Policy, + /** The default or active set of modules in the container has changed */ + Modules, + FolderType, + WebRoot, + AttachmentDirectory, + PipelineRoot, + Title, + Description, + SiteRoot, + StudyChange, + EndpointDirectory, + CloudStores + } + + static Path makePath(Container parent, String name) + { + if (null == parent) + return new Path(name); + return parent.getParsedPath().append(name, true); + } + + public static Container createMockContainer() + { + return new Container(null, "MockContainer", "01234567-8901-2345-6789-012345678901", 99999999, 0, new Date(), User.guest.getUserId(), true); + } + + private static Container createRoot() + { + Map m = new HashMap<>(); + m.put("Parent", null); + m.put("Name", ""); + Table.insert(null, CORE.getTableInfoContainers(), m); + + return getRoot(); + } + + private static DbScope.Transaction ensureTransaction() + { + return CORE.getSchema().getScope().ensureTransaction(DATABASE_QUERY_LOCK); + } + + private static int getNewChildSortOrder(Container parent) + { + int nextSortOrderVal = 0; + + List children = parent.getChildren(); + if (children != null) + { + for (Container child : children) + { + // find the max sort order value for the set of children + nextSortOrderVal = Math.max(nextSortOrderVal, child.getSortOrder()); + } + } + + // custom sorting applies: put new container at the end. + if (nextSortOrderVal > 0) + return nextSortOrderVal + 1; + + // we're sorted alphabetically + return 0; + } + + // TODO: Make private and force callers to use ensureContainer instead? + // TODO: Handle root creation here? + @NotNull + public static Container createContainer(Container parent, String name, @NotNull User user) + { + return createContainer(parent, name, null, null, NormalContainerType.NAME, user, null, null); + } + + public static final String WORKBOOK_DBSEQUENCE_NAME = "org.labkey.api.data.Workbooks"; + + // TODO: Pass in FolderType (separate from the container type of workbook, etc) and transact it with container creation? + @NotNull + public static Container createContainer(Container parent, String name, @Nullable String title, @Nullable String description, String type, @NotNull User user) + { + return createContainer(parent, name, title, description, type, user, null, null); + } + + @NotNull + public static Container createContainer(Container parent, String name, @Nullable String title, @Nullable String description, String type, @NotNull User user, @Nullable String auditMsg) + { + return createContainer(parent, name, title, description, type, user, auditMsg, null); + } + + @NotNull + public static Container createContainer(Container parent, String name, @Nullable String title, @Nullable String description, String type, @NotNull User user, @Nullable String auditMsg, + Consumer configureContainer) + { + ContainerType cType = ContainerTypeRegistry.get().getType(type); + if (cType == null) + throw new IllegalArgumentException("Unknown container type: " + type); + + // TODO: move this to ContainerType? + long sortOrder; + if (cType instanceof WorkbookContainerType) + { + sortOrder = DbSequenceManager.get(parent, WORKBOOK_DBSEQUENCE_NAME).next(); + + // Default workbook names are simply "" + if (name == null) + name = String.valueOf(sortOrder); + } + else + { + sortOrder = getNewChildSortOrder(parent); + } + + if (!parent.canHaveChildren()) + throw new IllegalArgumentException("Parent of a container must not be a " + parent.getContainerType().getName()); + + StringBuilder error = new StringBuilder(); + if (!Container.isLegalName(name, parent.isRoot(), error)) + throw new ApiUsageException(error.toString()); + + if (!Container.isLegalTitle(title, error)) + throw new ApiUsageException(error.toString()); + + Path path = makePath(parent, name); + SQLException sqlx = null; + Map insertMap = null; + + GUID entityId = new GUID(); + Container c; + + try + { + _constructing.add(entityId); + + try + { + Map m = new CaseInsensitiveHashMap<>(); + m.put("Parent", parent.getId()); + m.put("Name", name); + m.put("Title", title); + m.put("SortOrder", sortOrder); + m.put("EntityId", entityId); + if (null != description) + m.put("Description", description); + m.put("Type", type); + insertMap = Table.insert(user, CORE.getTableInfoContainers(), m); + } + catch (RuntimeSQLException x) + { + if (!x.isConstraintException()) + throw x; + sqlx = x.getSQLException(); + } + + _clearChildrenFromCache(parent); + + c = insertMap == null ? null : getForId(entityId); + + if (null == c) + { + if (null != sqlx) + throw new RuntimeSQLException(sqlx); + else + throw new RuntimeException("Container for path '" + path + "' was not created properly."); + } + + User savePolicyUser = user; + if (c.isProject() && !c.hasPermission(user, AdminPermission.class) && ContainerManager.getRoot().hasPermission(user, CreateProjectPermission.class)) + { + // Special case for project creators who don't necessarily yet have permission to save the policy of + // the project they just created + savePolicyUser = User.getAdminServiceUser(); + } + + // Workbooks inherit perms from their parent so don't create a policy if this is a workbook + if (c.isContainerFor(ContainerType.DataType.permissions)) + { + SecurityManager.setAdminOnlyPermissions(c, savePolicyUser); + } + + _removeFromCache(c, true); // seems odd, but it removes c.getProject() which clears other things from the cache + + // Initialize the list of active modules in the Container + c.getActiveModules(true, true, user); + + if (c.isProject()) + { + SecurityManager.createNewProjectGroups(c, savePolicyUser); + } + else + { + // If current user does NOT have admin permission on this container or the project has been + // explicitly set to have new subfolders inherit permissions, then inherit permissions + // (otherwise they would not be able to see the folder) + boolean hasAdminPermission = c.hasPermission(user, AdminPermission.class); + if ((!hasAdminPermission && !user.hasRootAdminPermission()) || SecurityManager.shouldNewSubfoldersInheritPermissions(c.getProject())) + SecurityManager.setInheritPermissions(c); + } + + // NOTE parent caches some info about children (e.g. hasWorkbookChildren) + // since mutating cached objects is frowned upon, just uncache parent + // CONSIDER: we could perhaps only uncache if the child is a workbook, but I think this reasonable + _removeFromCache(parent, true); + + if (null != configureContainer) + configureContainer.accept(c); + } + finally + { + _constructing.remove(entityId); + } + + fireCreateContainer(c, user, auditMsg); + + return c; + } + + public static void addSecurableResourceProvider(ContainerSecurableResourceProvider provider) + { + _resourceProviders.add(provider); + } + + public static List getSecurableResourceProviders() + { + return Collections.unmodifiableList(_resourceProviders); + } + + public static Container createContainerFromTemplate(Container parent, String name, String title, Container templateContainer, User user, FolderExportContext exportCtx, Consumer afterCreateHandler) throws Exception + { + MemoryVirtualFile vf = new MemoryVirtualFile(); + + // export objects from the source template folder + FolderWriterImpl writer = new FolderWriterImpl(); + writer.write(templateContainer, exportCtx, vf); + + // create the new target container + Container c = createContainer(parent, name, title, null, NormalContainerType.NAME, user, null, afterCreateHandler); + + // import objects into the target folder + XmlObject folderXml = vf.getXmlBean("folder.xml"); + if (folderXml instanceof FolderDocument folderDoc) + { + FolderImportContext importCtx = new FolderImportContext(user, c, folderDoc, null, new StaticLoggerGetter(LogManager.getLogger(FolderImporterImpl.class)), vf); + + FolderImporterImpl importer = new FolderImporterImpl(); + importer.process(null, importCtx, vf); + } + + return c; + } + + public static void setRequireAuditComments(Container container, User user, @NotNull Boolean required) + { + WritablePropertyMap props = PropertyManager.getWritableProperties(container, AUDIT_SETTINGS_PROPERTY_SET_NAME, true); + String originalValue = props.get(REQUIRE_USER_COMMENTS_PROPERTY_NAME); + props.put(REQUIRE_USER_COMMENTS_PROPERTY_NAME, required.toString()); + props.save(); + + addAuditEvent(user, container, + "Changed " + REQUIRE_USER_COMMENTS_PROPERTY_NAME + " from \"" + + originalValue + "\" to \"" + required + "\""); + } + + public static void setFolderType(Container c, FolderType folderType, User user, BindException errors) + { + FolderType oldType = c.getFolderType(); + + if (folderType.equals(oldType)) + return; + + List errorStrings = new ArrayList<>(); + + if (!c.isProject() && folderType.isProjectOnlyType()) + errorStrings.add("Cannot set a subfolder to " + folderType.getName() + " because it is a project-only folder type."); + + // Check for any containers that need to be moved into container tabs + if (errorStrings.isEmpty() && folderType.hasContainerTabs()) + { + List childTabFoldersNonMatchingTypes = new ArrayList<>(); + List containersBecomingTabs = findAndCheckContainersMatchingTabs(c, folderType, childTabFoldersNonMatchingTypes, errorStrings); + + if (errorStrings.isEmpty()) + { + if (!containersBecomingTabs.isEmpty()) + { + // Make containers tab container; Folder tab will find them by name + try (DbScope.Transaction transaction = ensureTransaction()) + { + for (Container container : containersBecomingTabs) + updateType(container, TabContainerType.NAME, user); + + transaction.commit(); + } + } + + // Check these and change type unless they were overridden explicitly + for (Container container : childTabFoldersNonMatchingTypes) + { + if (!isContainerTabTypeOverridden(container)) + { + FolderTab newTab = folderType.findTab(container.getName()); + assert null != newTab; // There must be a tab because it caused the container to get into childTabFoldersNonMatchingTypes + FolderType newType = newTab.getFolderType(); + if (null == newType) + newType = FolderType.NONE; // default to NONE + setFolderType(container, newType, user, errors); + } + } + } + } + + if (errorStrings.isEmpty()) + { + oldType.unconfigureContainer(c, user); + WritablePropertyMap props = PropertyManager.getWritableProperties(c, FOLDER_TYPE_PROPERTY_SET_NAME, true); + props.put(FOLDER_TYPE_PROPERTY_NAME, folderType.getName()); + + if (c.isContainerTab()) + { + boolean containerTabTypeOverridden = false; + FolderTab tab = c.getParent().getFolderType().findTab(c.getName()); + if (null != tab && !folderType.equals(tab.getFolderType())) + containerTabTypeOverridden = true; + props.put(FOLDER_TYPE_PROPERTY_TABTYPE_OVERRIDDEN, Boolean.toString(containerTabTypeOverridden)); + } + props.save(); + + notifyContainerChange(c.getId(), Property.FolderType, user); + folderType.configureContainer(c, user); // Configure new only after folder type has been changed + + // TODO: Not needed? I don't think we've changed the container's state. + _removeFromCache(c, false); + } + else + { + for (String errorString : errorStrings) + errors.reject(SpringActionController.ERROR_MSG, errorString); + } + } + + public static void checkContainerValidity(Container c) throws ContainerException + { + // Check container for validity; in rare cases user may have changed their custom folderType.xml and caused + // duplicate subfolders (same name) to exist + // Get list of child containers that are not container tabs, but match container tabs; these are bad + FolderType folderType = getFolderType(c); + List errorStrings = new ArrayList<>(); + List childTabFoldersNonMatchingTypes = new ArrayList<>(); + List containersMatchingTabs = findAndCheckContainersMatchingTabs(c, folderType, childTabFoldersNonMatchingTypes, errorStrings); + if (!containersMatchingTabs.isEmpty()) + { + throw new Container.ContainerException("Folder " + c.getPath() + + " has a subfolder with the same name as a container tab folder, which is an invalid state." + + " This may have been caused by changing the folder type's tabs after this folder was set to its folder type." + + " An administrator should either delete the offending subfolder or change the folder's folder type.\n"); + } + } + + public static List findAndCheckContainersMatchingTabs(Container c, FolderType folderType, + List childTabFoldersNonMatchingTypes, List errorStrings) + { + List containersMatchingTabs = new ArrayList<>(); + for (FolderTab folderTab : folderType.getDefaultTabs()) + { + if (folderTab.getTabType() == FolderTab.TAB_TYPE.Container) + { + for (Container child : c.getChildren()) + { + if (child.getName().equalsIgnoreCase(folderTab.getName())) + { + if (!child.getFolderType().getName().equalsIgnoreCase(folderTab.getFolderTypeName())) + { + if (child.isContainerTab()) + childTabFoldersNonMatchingTypes.add(child); // Tab type doesn't match child tab folder + else + errorStrings.add("Child folder " + child.getName() + + " matches container tab, but folder type " + child.getFolderType().getName() + " doesn't match tab's folder type " + + folderTab.getFolderTypeName() + "."); + } + + int childCount = child.getChildren().size(); + if (childCount > 0) + { + errorStrings.add("Child folder " + child.getName() + + " matches container tab, but cannot be converted to a tab folder because it has " + childCount + " children."); + } + + if (!child.isConvertibleToTab()) + { + errorStrings.add("Child folder " + child.getName() + + " matches container tab, but cannot be converted to a tab folder because it is a " + child.getContainerNoun() + "."); + } + + if (!child.isContainerTab()) + containersMatchingTabs.add(child); + + break; // we found name match; can't be another + } + } + } + } + return containersMatchingTabs; + } + + private static final Set containersWithBadFolderTypes = new ConcurrentHashSet<>(); + + @NotNull + public static FolderType getFolderType(Container c) + { + String name = getFolderTypeName(c); + FolderType folderType; + + if (null != name) + { + folderType = FolderTypeManager.get().getFolderType(name); + + if (null == folderType) + { + // If we're upgrading then folder types won't be defined yet... don't warn in that case. + if (!ModuleLoader.getInstance().isUpgradeInProgress() && + !ModuleLoader.getInstance().isUpgradeRequired() && + !containersWithBadFolderTypes.contains(c)) + { + LOG.warn("No such folder type " + name + " for folder " + c.toString()); + containersWithBadFolderTypes.add(c); + } + + folderType = FolderType.NONE; + } + } + else + folderType = FolderType.NONE; + + return folderType; + } + + /** + * Most code should call getFolderType() instead. + * Useful for finding the name of the folder type BEFORE startup is complete, so the FolderType itself + * may not be available. + */ + @Nullable + public static String getFolderTypeName(Container c) + { + Map props = PropertyManager.getProperties(c, FOLDER_TYPE_PROPERTY_SET_NAME); + return props.get(FOLDER_TYPE_PROPERTY_NAME); + } + + + @NotNull + public static Map getFolderTypeNameContainerCounts(Container root) + { + Map nameCounts = new TreeMap<>(); + for (Container c : getAllChildren(root)) + { + Integer count = nameCounts.get(c.getFolderType().getName()); + if (null == count) + { + count = Integer.valueOf(0); + } + nameCounts.put(c.getFolderType().getName(), ++count); + } + return nameCounts; + } + + @NotNull + public static Map getProductFoldersMetrics(@NotNull FolderType folderType) + { + Container root = getRoot(); + Map metrics = new TreeMap<>(); + List counts = new ArrayList<>(); + for (Container c : root.getChildren()) + { + if (!c.getFolderType().getName().equals(folderType.getName())) + continue; + + int childCount = c.getChildren().stream().filter(Container::isInFolderNav).toList().size(); + counts.add(childCount); + } + + int totalFolderTypeMatch = counts.size(); + if (totalFolderTypeMatch == 0) + return metrics; + + Collections.sort(counts); + int median = counts.get((totalFolderTypeMatch - 1)/2); + if (totalFolderTypeMatch % 2 == 0 ) + { + int low = counts.get(totalFolderTypeMatch/2 - 1); + int high = counts.get(totalFolderTypeMatch/2); + median = Math.round((low + high) / 2.0f); + } + int maxProjectsCount = counts.get(totalFolderTypeMatch - 1); + int totalProjectsCount = counts.stream().mapToInt(Integer::intValue).sum(); + int averageProjectsCount = Math.round((float) totalProjectsCount /totalFolderTypeMatch); + + metrics.put("totalSubProjectsCount", totalProjectsCount); + metrics.put("averageSubProjectsPerHomeProject", averageProjectsCount); + metrics.put("medianSubProjectsCountPerHomeProject", median); + metrics.put("maxSubProjectsCountInHomeProject", maxProjectsCount); + + return metrics; + } + + public static boolean isContainerTabTypeThisOrChildrenOverridden(Container c) + { + if (isContainerTabTypeOverridden(c)) + return true; + if (c.getFolderType().hasContainerTabs()) + { + for (Container child : c.getChildren()) + { + if (child.isContainerTab() && isContainerTabTypeOverridden(child)) + return true; + } + } + return false; + } + + public static boolean isContainerTabTypeOverridden(Container c) + { + Map props = PropertyManager.getProperties(c, FOLDER_TYPE_PROPERTY_SET_NAME); + String overridden = props.get(FOLDER_TYPE_PROPERTY_TABTYPE_OVERRIDDEN); + return (null != overridden) && overridden.equalsIgnoreCase("true"); + } + + private static void setContainerTabDeleted(Container c, String tabName, String folderTypeName) + { + // Add prop in this category + WritablePropertyMap props = PropertyManager.getWritableProperties(c, TABFOLDER_CHILDREN_DELETED, true); + props.put(getDeletedTabKey(tabName, folderTypeName), "true"); + props.save(); + } + + public static void clearContainerTabDeleted(Container c, String tabName, String folderTypeName) + { + WritablePropertyMap props = PropertyManager.getWritableProperties(c, TABFOLDER_CHILDREN_DELETED, true); + String key = getDeletedTabKey(tabName, folderTypeName); + if (props.containsKey(key)) + { + props.remove(key); + props.save(); + } + } + + public static boolean hasContainerTabBeenDeleted(Container c, String tabName, String folderTypeName) + { + // We keep arbitrary number of deleted children tabs using suffix 0, 1, 2.... + Map props = PropertyManager.getProperties(c, TABFOLDER_CHILDREN_DELETED); + return props.containsKey(getDeletedTabKey(tabName, folderTypeName)); + } + + private static String getDeletedTabKey(String tabName, String folderTypeName) + { + return tabName + "-TABDELETED-FOLDER-" + folderTypeName; + } + + @NotNull + public static Container ensureContainer(@NotNull String path, @NotNull User user) + { + return ensureContainer(Path.parse(path), user); + } + + @NotNull + public static Container ensureContainer(@NotNull Path path, @NotNull User user) + { + Container c = null; + + try + { + c = getForPath(path); + } + catch (RootContainerException e) + { + // Ignore this -- root doesn't exist yet + } + + if (null == c) + { + if (path.isEmpty()) + c = createRoot(); + else + { + Path parentPath = path.getParent(); + c = ensureContainer(parentPath, user); + c = createContainer(c, path.getName(), null, null, NormalContainerType.NAME, user); + } + } + return c; + } + + + @NotNull + public static Container ensureContainer(Container parent, String name, User user) + { + // NOTE: Running outside a tx doesn't seem to be necessary. +// if (CORE.getSchema().getScope().isTransactionActive()) +// throw new IllegalStateException("Transaction should not be active"); + + Container c = null; + + try + { + c = getForPath(makePath(parent,name)); + } + catch (RootContainerException e) + { + // Ignore this -- root doesn't exist yet + } + + if (null == c) + { + c = createContainer(parent, name, user); + } + return c; + } + + public static void updateDescription(Container container, String description, User user) + throws ValidationException + { + ColumnValidators.validate(CORE.getTableInfoContainers().getColumn("Title"), null, 1, description); + + //For some reason, there is no primary key defined on core.containers + //so we can't use Table.update here + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + sql.append(" SET Description=? WHERE RowID=?").add(description).add(container.getRowId()); + new SqlExecutor(CORE.getSchema()).execute(sql); + + String oldValue = container.getDescription(); + _removeFromCache(container, false); + container = getForRowId(container.getRowId()); + ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(container, user, Property.Description, oldValue, description); + firePropertyChangeEvent(evt); + } + + public static void updateSearchable(Container container, boolean searchable, User user) + { + //For some reason, there is no primary key defined on core.containers + //so we can't use Table.update here + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + sql.append(" SET Searchable=? WHERE RowID=?").add(searchable).add(container.getRowId()); + new SqlExecutor(CORE.getSchema()).execute(sql); + + _removeFromCache(container, false); + } + + public static void updateLockState(Container container, LockState lockState, @NotNull Runnable auditRunnable) + { + //For some reason there is no primary key defined on core.containers + //so we can't use Table.update here + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + sql.append(" SET LockState = ?, ExpirationDate = NULL WHERE RowID = ?").add(lockState).add(container.getRowId()); + new SqlExecutor(CORE.getSchema()).execute(sql); + + _removeFromCache(container, false); + + auditRunnable.run(); + } + + public static List getExcludedProjects() + { + return getProjects().stream() + .filter(p->p.getLockState() == Container.LockState.Excluded) + .collect(Collectors.toList()); + } + + public static List getNonExcludedProjects() + { + return getProjects().stream() + .filter(p->p.getLockState() != Container.LockState.Excluded) + .collect(Collectors.toList()); + } + + public static void setExcludedProjects(Collection ids, @NotNull Runnable auditRunnable) + { + // First clear all existing "Excluded" states + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + sql.append(" SET LockState = NULL, ExpirationDate = NULL WHERE LockState = ?").add(LockState.Excluded); + new SqlExecutor(CORE.getSchema()).execute(sql); + + // Now set the passed-in projects to "Excluded" + if (!ids.isEmpty()) + { + ColumnInfo entityIdCol = CORE.getTableInfoContainers().getColumn("EntityId"); + Filter inClauseFilter = new SimpleFilter(new InClause(entityIdCol.getFieldKey(), ids)); + SQLFragment frag = new SQLFragment("UPDATE "); + frag.append(CORE.getTableInfoContainers().getSelectName()); + frag.append(" SET LockState = ?, ExpirationDate = NULL "); + frag.add(LockState.Excluded); + frag.append(inClauseFilter.getSQLFragment(CORE.getSqlDialect(), "c", Map.of(entityIdCol.getFieldKey(), entityIdCol))); + new SqlExecutor(CORE.getSchema()).execute(frag); + } + + clearCache(); + + auditRunnable.run(); + } + + public static void archiveContainer(User user, Container container, boolean archive) + { + if (container.isRoot() || container.isProject() || container.isAppHomeFolder()) + throw new ApiUsageException("Archive action not supported for this folder."); + + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers().getSelectName()); + if (archive) + { + sql.append(" SET LockState = ? "); + sql.add(LockState.Archived); + sql.append(" WHERE LockState IS NULL "); + } + else + { + sql.append(" SET LockState = NULL WHERE LockState = ? "); + sql.add(LockState.Archived); + } + sql.append("AND EntityId = ? "); + sql.add(container.getEntityId()); + new SqlExecutor(CORE.getSchema()).execute(sql); + + clearCache(); + + addAuditEvent(user, container, archive ? "Container has been archived." : "Archived container has been restored."); + } + + public static void updateExpirationDate(Container container, LocalDate expirationDate, @NotNull Runnable auditRunnable) + { + //For some reason there is no primary key defined on core.containers + //so we can't use Table.update here + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + // Note: jTDS doesn't support LocalDate, so convert to java.sql.Date + sql.append(" SET ExpirationDate = ? WHERE RowID = ?").add(java.sql.Date.valueOf(expirationDate)).add(container.getRowId()); + + new SqlExecutor(CORE.getSchema()).execute(sql); + + _removeFromCache(container, false); + + auditRunnable.run(); + } + + public static void updateType(Container container, String newType, User user) + { + //For some reason there is no primary key defined on core.containers + //so we can't use Table.update here + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + sql.append(" SET Type=? WHERE RowID=?").add(newType).add(container.getRowId()); + new SqlExecutor(CORE.getSchema()).execute(sql); + + _removeFromCache(container, false); + } + + public static void updateTitle(Container container, String title, User user) + throws ValidationException + { + ColumnValidators.validate(CORE.getTableInfoContainers().getColumn("Title"), null, 1, title); + + //For some reason there is no primary key defined on core.containers + //so we can't use Table.update here + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + sql.append(" SET Title=? WHERE RowID=?").add(title).add(container.getRowId()); + new SqlExecutor(CORE.getSchema()).execute(sql); + + _removeFromCache(container, false); + String oldValue = container.getTitle(); + container = getForRowId(container.getRowId()); + ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(container, user, Property.Title, oldValue, title); + firePropertyChangeEvent(evt); + } + + public static void uncache(Container c) + { + _removeFromCache(c, true); + } + + public static final String SHARED_CONTAINER_PATH = "/Shared"; + + @NotNull + public static Container getSharedContainer() + { + return ensureContainer(Path.parse(SHARED_CONTAINER_PATH), User.getAdminServiceUser()); + } + + public static List getChildren(Container parent) + { + return new ArrayList<>(getChildrenMap(parent).values()); + } + + // Default is to include all types of children, as seems only appropriate + public static List getChildren(Container parent, User u, Class perm) + { + return getChildren(parent, u, perm, null, ContainerTypeRegistry.get().getTypeNames()); + } + + public static List getChildren(Container parent, User u, Class perm, Set roles) + { + return getChildren(parent, u, perm, roles, ContainerTypeRegistry.get().getTypeNames()); + } + + public static List getChildren(Container parent, User u, Class perm, String typeIncluded) + { + return getChildren(parent, u, perm, null, Collections.singleton(typeIncluded)); + } + + public static List getChildren(Container parent, User u, Class perm, Set roles, Set includedTypes) + { + List children = new ArrayList<>(); + for (Container child : getChildrenMap(parent).values()) + if (includedTypes.contains(child.getContainerType().getName()) && child.hasPermission(u, perm, roles)) + children.add(child); + + return children; + } + + public static List getAllChildren(Container parent, User u) + { + return getAllChildren(parent, u, ReadPermission.class, null, ContainerTypeRegistry.get().getTypeNames()); + } + + public static List getAllChildren(Container parent, User u, Class perm) + { + return getAllChildren(parent, u, perm, null, ContainerTypeRegistry.get().getTypeNames()); + } + + // Default is to include all types of children + public static List getAllChildren(Container parent, User u, Class perm, Set roles) + { + return getAllChildren(parent, u, perm, roles, ContainerTypeRegistry.get().getTypeNames()); + } + + public static List getAllChildren(Container parent, User u, Class perm, String typeIncluded) + { + return getAllChildren(parent, u, perm, null, Collections.singleton(typeIncluded)); + } + + public static List getAllChildren(Container parent, User u, Class perm, Set roles, Set typesIncluded) + { + Set allChildren = getAllChildren(parent); + List result = new ArrayList<>(allChildren.size()); + + for (Container container : allChildren) + { + if (typesIncluded.contains(container.getContainerType().getName()) && container.hasPermission(u, perm, roles)) + { + result.add(container); + } + } + + return result; + } + + // Returns the next available child container name based on the baseName + public static String getAvailableChildContainerName(Container c, String baseName) + { + List children = getChildren(c); + Map folders = new HashMap<>(children.size() * 2); + for (Container child : children) + folders.put(child.getName(), child); + + String availableContainerName = baseName; + int i = 1; + while (folders.containsKey(availableContainerName)) + { + availableContainerName = baseName + " " + i++; + } + + return availableContainerName; + } + + // Returns true only if user has the specified permission in the entire container tree starting at root + public static boolean hasTreePermission(Container root, User u, Class perm) + { + for (Container c : getAllChildren(root)) + if (!c.hasPermission(u, perm)) + return false; + + return true; + } + + private static Map getChildrenMap(Container parent) + { + if (!parent.canHaveChildren()) + { + // Optimization to avoid database query (important because some installs have tens of thousands of + // workbooks) when the container is a workbook, which is not allowed to have children + return Collections.emptyMap(); + } + + List childIds = CACHE_CHILDREN.get(parent.getEntityId()); + if (null == childIds) + { + try (DbScope.Transaction t = ensureTransaction()) + { + List children = new SqlSelector(CORE.getSchema(), + "SELECT * FROM " + CORE.getTableInfoContainers() + " WHERE Parent = ? ORDER BY SortOrder, LOWER(Name)", + parent.getId()).getArrayList(Container.class); + + childIds = new ArrayList<>(children.size()); + for (Container c : children) + { + childIds.add(c.getEntityId()); + _addToCache(c); + } + childIds = Collections.unmodifiableList(childIds); + CACHE_CHILDREN.put(parent.getEntityId(), childIds); + // No database changes to commit, but need to decrement the transaction counter + t.commit(); + } + } + + if (childIds.isEmpty()) + return Collections.emptyMap(); + + // Use a LinkedHashMap to preserve the order defined by the user - they're not necessarily alphabetical + Map ret = new LinkedHashMap<>(); + for (GUID id : childIds) + { + Container c = getForId(id); + if (null != c) + ret.put(c.getName(), c); + } + return Collections.unmodifiableMap(ret); + } + + public static Container getForRowId(int id) + { + Selector selector = new SqlSelector(CORE.getSchema(), new SQLFragment("SELECT * FROM " + CORE.getTableInfoContainers() + " WHERE RowId = ?", id)); + return selector.getObject(Container.class); + } + + public static @Nullable Container getForId(@NotNull GUID guid) + { + return guid != null ? getForId(guid.toString()) : null; + } + + public static @Nullable Container getForId(@Nullable String id) + { + //if the input string is not a GUID, just return null, + //so that we don't get a SQLException when the database + //tries to convert it to a unique identifier. + if (!GUID.isGUID(id)) + return null; + + GUID guid = new GUID(id); + + Container d = CACHE_ENTITY_ID.get(guid); + if (null != d) + return d; + + try (DbScope.Transaction t = ensureTransaction()) + { + Container result = new SqlSelector( + CORE.getSchema(), + "SELECT * FROM " + CORE.getTableInfoContainers() + " WHERE EntityId = ?", + id).getObject(Container.class); + if (result != null) + { + result = _addToCache(result); + } + // No database changes to commit, but need to decrement the counter + t.commit(); + + return result; + } + } + + public static Container getChild(Container c, String name) + { + Path path = c.getParsedPath().append(name); + + Container d = _getFromCachePath(path); + if (null != d) + return d; + + Map map = getChildrenMap(c); + return map.get(name); + } + + + public static Container getForURL(@NotNull ActionURL url) + { + Container ret = getForPath(url.getExtraPath()); + if (ret == null) + ret = getForId(StringUtils.strip(url.getExtraPath(), "/")); + return ret; + } + + + public static Container getForPath(@NotNull String path) + { + if (GUID.isGUID(path)) + { + Container c = getForId(path); + if (c != null) + return c; + } + + Path p = Path.parse(path); + return getForPath(p); + } + + public static Container getForPath(Path path) + { + Container d = _getFromCachePath(path); + if (null != d) + return d; + + // Special case for ROOT -- we want to throw instead of returning null + if (path.equals(Path.rootPath)) + { + try (DbScope.Transaction t = ensureTransaction()) + { + TableInfo tinfo = CORE.getTableInfoContainers(); + + // Unusual, but possible -- if cache loader hits an exception it can end up caching null + if (null == tinfo) + throw new RootContainerException("Container table could not be retrieved from the cache"); + + // This might be called at bootstrap, before schemas have been created + if (tinfo.getTableType() == DatabaseTableType.NOT_IN_DB) + throw new RootContainerException("Container table has not been created"); + + Container result = new SqlSelector(CORE.getSchema(),"SELECT * FROM " + tinfo + " WHERE Parent IS NULL").getObject(Container.class); + + if (result == null) + throw new RootContainerException("Root container does not exist"); + + _addToCache(result); + // No database changes to commit, but need to decrement the counter + t.commit(); + return result; + } + } + else + { + Path parent = path.getParent(); + String name = path.getName(); + Container dirParent = getForPath(parent); + + if (null == dirParent) + return null; + + Map map = getChildrenMap(dirParent); + return map.get(name); + } + } + + public static class RootContainerException extends RuntimeException + { + private RootContainerException(String message, Throwable cause) + { + super(message, cause); + } + + private RootContainerException(String message) + { + super(message); + } + } + + public static Container getRoot() + { + try + { + return getForPath("/"); + } + catch (MinorConfigurationException e) + { + // If the server is misconfigured, rethrow so some callers don't swallow it and other callers don't end up + // reporting it to mothership, Issue 50843. + throw e; + } + catch (Exception e) + { + // Some callers catch and ignore this exception, e.g., early in the bootstrap process + throw new RootContainerException("Root container can't be retrieved", e); + } + } + + public static void saveAliasesForContainer(Container container, List aliases, User user) + { + Set originalAliases = new CaseInsensitiveHashSet(getAliasesForContainer(container)); + Set newAliases = new CaseInsensitiveHashSet(aliases); + + if (originalAliases.equals(newAliases)) + { + return; + } + + try (DbScope.Transaction transaction = ensureTransaction()) + { + // Delete all of the aliases for the current container, plus any of the aliases that might be associated + // with another container right now + SQLFragment deleteSQL = new SQLFragment(); + deleteSQL.append("DELETE FROM "); + deleteSQL.append(CORE.getTableInfoContainerAliases()); + deleteSQL.append(" WHERE ContainerRowId = ? "); + deleteSQL.add(container.getRowId()); + if (!aliases.isEmpty()) + { + deleteSQL.append(" OR Path IN ("); + String separator = ""; + for (String alias : aliases) + { + deleteSQL.append(separator); + separator = ", "; + deleteSQL.append("LOWER(?)"); + deleteSQL.add(alias); + } + deleteSQL.append(")"); + } + new SqlExecutor(CORE.getSchema()).execute(deleteSQL); + + // Store the alias as LOWER() so that we can query against it using the index + for (String alias : newAliases) + { + SQLFragment insertSQL = new SQLFragment(); + insertSQL.append("INSERT INTO "); + insertSQL.append(CORE.getTableInfoContainerAliases()); + insertSQL.append(" (Path, ContainerRowId) VALUES (LOWER(?), ?)"); + insertSQL.add(alias); + insertSQL.add(container.getRowId()); + new SqlExecutor(CORE.getSchema()).execute(insertSQL); + } + + addAuditEvent(user, container, + "Changed folder aliases from \"" + + StringUtils.join(originalAliases, ", ") + "\" to \"" + + StringUtils.join(newAliases, ", ") + "\""); + + transaction.commit(); + } + } + + // Abstract base class used for attaching system resources (favorite icons, logos, stylesheets, sso auth logos) to folders and projects + public static abstract class ContainerParent implements AttachmentParent + { + private final Container _c; + + protected ContainerParent(Container c) + { + _c = c; + } + + @Override + public String getEntityId() + { + return _c.getId(); + } + + @Override + public String getContainerId() + { + return _c.getId(); + } + + public Container getContainer() + { + return _c; + } + } + + public static Container getHomeContainer() + { + return getForPath(HOME_PROJECT_PATH); + } + + public static List getProjects() + { + return getChildren(getRoot()); + } + + public static NavTree getProjectList(ViewContext context, boolean includeChildren) + { + User user = context.getUser(); + Container currentProject = context.getContainer().getProject(); + String projectNavTreeId = PROJECT_LIST_ID; + if (currentProject != null) + projectNavTreeId += currentProject.getId(); + + NavTree navTree = (NavTree) NavTreeManager.getFromCache(projectNavTreeId, context); + if (null != navTree) + return navTree; + + NavTree list = new NavTree("Projects"); + List projects = getProjects(); + + for (Container project : projects) + { + boolean shouldDisplay = project.shouldDisplay(user) && project.hasPermission("getProjectList()", user, ReadPermission.class); + boolean includeCurrentProject = includeChildren && currentProject != null && currentProject.equals(project); + + if (shouldDisplay || includeCurrentProject) + { + ActionURL startURL = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(project); + + if (includeChildren) + list.addChild(getFolderListForUser(project, context)); + else if (project.equals(getHomeContainer())) + list.addChild(new NavTree("Home", startURL)); + else + list.addChild(project.getTitle(), startURL); + } + } + + list.setId(projectNavTreeId); + NavTreeManager.cacheTree(list, context.getUser()); + + return list; + } + + public static NavTree getFolderListForUser(final Container project, ViewContext viewContext) + { + final boolean isNavAccessOpen = AppProps.getInstance().isNavigationAccessOpen(); + final Container c = viewContext.getContainer(); + final String cacheKey = isNavAccessOpen ? project.getId() : c.getId(); + + NavTree tree = (NavTree) NavTreeManager.getFromCache(cacheKey, viewContext); + if (null != tree) + return tree; + + try + { + assert SecurityLogger.indent("getFolderListForUser()"); + + User user = viewContext.getUser(); + String projectId = project.getId(); + + List folders = new ArrayList<>(getAllChildren(project)); + + Collections.sort(folders); + + Set containersInTree = new HashSet<>(); + + Map m = new HashMap<>(); + Map permission = new HashMap<>(); + + for (Container f : folders) + { + if (!f.isInFolderNav()) + continue; + + boolean hasPolicyRead = f.hasPermission(user, ReadPermission.class); + + boolean skip = ( + !hasPolicyRead || + !f.shouldDisplay(user) || + !f.hasPermission(user, ReadPermission.class) + ); + + //Always put the project and current container in... + if (skip && !f.equals(project) && !f.equals(c)) + continue; + + //HACK to make home link consistent... + String name = f.getTitle(); + if (name.equals("home") && f.equals(getHomeContainer())) + name = "Home"; + + NavTree t = new NavTree(name); + + // 34137: Support folder path expansion for containers where label != name + t.setId(f.getId()); + if (hasPolicyRead) + { + ActionURL url = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(f); + t.setHref(url.getEncodedLocalURIString()); + } + + boolean addFolder = false; + + if (isNavAccessOpen) + { + addFolder = true; + } + else + { + // 32718: If navigation access is not open then hide projects that aren't directly + // accessible in site folder navigation. + + if (f.equals(c) || f.isRoot() || (hasPolicyRead && f.isProject())) + { + // In current container, root, or readable project + addFolder = true; + } + else + { + boolean isAscendant = f.isDescendant(c); + boolean isDescendant = c.isDescendant(f); + boolean inActivePath = isAscendant || isDescendant; + boolean hasAncestryRead = false; + + if (inActivePath) + { + Container leaf = isAscendant ? f : c; + Container localRoot = isAscendant ? c : f; + + List ancestors = containersToRootList(leaf); + Collections.reverse(ancestors); + + for (Container p : ancestors) + { + if (!permission.containsKey(p.getId())) + permission.put(p.getId(), p.hasPermission(user, ReadPermission.class)); + boolean hasRead = permission.get(p.getId()); + + if (p.equals(localRoot)) + { + hasAncestryRead = hasRead; + break; + } + else if (!hasRead) + { + hasAncestryRead = false; + break; + } + } + } + else + { + hasAncestryRead = containersToRoot(f).stream().allMatch(p -> { + if (!permission.containsKey(p.getId())) + permission.put(p.getId(), p.hasPermission(user, ReadPermission.class)); + return permission.get(p.getId()); + }); + } + + if (hasPolicyRead && hasAncestryRead && inActivePath) + { + // Is in the direct readable lineage of the current container + addFolder = true; + } + else if (hasPolicyRead && f.getParent().equals(c.getParent())) + { + // Is a readable sibling of the current container + addFolder = true; + } + else if (hasAncestryRead) + { + // Is a part of a fully readable ancestry + addFolder = true; + } + } + + if (!addFolder) + LOG.debug("isNavAccessOpen restriction: \"" + f.getPath() + "\""); + } + + if (addFolder) + { + containersInTree.add(f); + m.put(f.getId(), t); + } + } + + //Ensure parents of any accessible folder are in the tree. If not add them with no link. + for (Container treeContainer : containersInTree) + { + if (!treeContainer.equals(project) && !containersInTree.contains(treeContainer.getParent())) + { + Set containersToRoot = containersToRoot(treeContainer); + //Possible will be added more than once, if several children are accessible, but that's OK... + for (Container missing : containersToRoot) + { + if (!m.containsKey(missing.getId())) + { + if (isNavAccessOpen) + { + NavTree noLinkTree = new NavTree(missing.getName()); + noLinkTree.setId(missing.getId()); + m.put(missing.getId(), noLinkTree); + } + else + { + if (!permission.containsKey(missing.getId())) + permission.put(missing.getId(), missing.hasPermission(user, ReadPermission.class)); + + if (!permission.get(missing.getId())) + { + NavTree noLinkTree = new NavTree(missing.getName()); + m.put(missing.getId(), noLinkTree); + } + else + { + NavTree linkTree = new NavTree(missing.getName()); + ActionURL url = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(missing); + linkTree.setHref(url.getEncodedLocalURIString()); + m.put(missing.getId(), linkTree); + } + } + } + } + } + } + + for (Container f : folders) + { + if (f.getId().equals(projectId)) + continue; + + NavTree child = m.get(f.getId()); + if (null == child) + continue; + + NavTree parent = m.get(f.getParent().getId()); + assert null != parent; //This should not happen anymore, we assure all parents are in tree. + if (null != parent) + parent.addChild(child); + } + + NavTree projectTree = m.get(projectId); + + projectTree.setId(cacheKey); + + NavTreeManager.cacheTree(projectTree, user); + return projectTree; + } + finally + { + assert SecurityLogger.outdent(); + } + } + + public static Set containersToRoot(Container child) + { + Set containersOnPath = new HashSet<>(); + Container current = child; + while (current != null && !current.isRoot()) + { + containersOnPath.add(current); + current = current.getParent(); + } + + return containersOnPath; + } + + /** + * Provides a sorted list of containers from the root to the child container provided. + * It does not include the root node. + * @param child Container from which the search is sourced. + * @return List sorted in order of distance from root. + */ + public static List containersToRootList(Container child) + { + List containers = new ArrayList<>(); + Container current = child; + while (current != null && !current.isRoot()) + { + containers.add(current); + current = current.getParent(); + } + + Collections.reverse(containers); + return containers; + } + + // Move a container to another part of the container tree. Careful: this method DOES NOT prevent you from orphaning + // an entire tree (e.g., by setting a container's parent to one of its children); the UI in AdminController does this. + // + // NOTE: Beware side-effect of changing ACLs and GROUPS if a container changes projects + // + // @return true if project has changed (should probably redirect to security page) + public static boolean move(Container c, final Container newParent, User user) throws ValidationException + { + if (!isRenameable(c)) + { + throw new IllegalArgumentException("Can't move container " + c.getPath()); + } + + try (QuietCloser ignored = lockForMutation(MutatingOperation.move, c)) + { + List errors = new ArrayList<>(); + for (ContainerListener listener : getListeners()) + { + try + { + errors.addAll(listener.canMove(c, newParent, user)); + } + catch (Exception e) + { + ExceptionUtil.logExceptionToMothership(null, new IllegalStateException(listener.getClass().getName() + ".canMove() threw an exception or violated @NotNull contract")); + } + } + if (!errors.isEmpty()) + { + ValidationException exception = new ValidationException(); + for (String error : errors) + { + exception.addError(new SimpleValidationError(error)); + } + throw exception; + } + + if (c.getParent().getId().equals(newParent.getId())) + return false; + + Container oldParent = c.getParent(); + Container oldProject = c.getProject(); + Container newProject = newParent.isRoot() ? c : newParent.getProject(); + + boolean changedProjects = !oldProject.getId().equals(newProject.getId()); + + // Synchronize the transaction, but not the listeners -- see #9901 + try (DbScope.Transaction t = ensureTransaction()) + { + new SqlExecutor(CORE.getSchema()).execute("UPDATE " + CORE.getTableInfoContainers() + " SET Parent = ? WHERE EntityId = ?", newParent.getId(), c.getId()); + + // Refresh the container directly from the database so the container reflects the new parent, isProject(), etc. + c = getForRowId(c.getRowId()); + + // this could be done in the trigger, but I prefer to put it in the transaction + if (changedProjects) + SecurityManager.changeProject(c, oldProject, newProject, user); + + clearCache(); + + try + { + ExperimentService.get().moveContainer(c, oldParent, newParent); + } + catch (ExperimentException e) + { + throw new RuntimeException(e); + } + + // Clear after the commit has propagated the state to other threads and transactions + // Do this in a commit task in case we've joined another existing DbScope.Transaction instead of starting our own + t.addCommitTask(() -> + { + clearCache(); + getChildrenMap(newParent); // reload the cache + }, DbScope.CommitTaskOption.POSTCOMMIT); + + t.commit(); + } + + Container newContainer = getForId(c.getId()); + fireMoveContainer(newContainer, oldParent, user); + + return changedProjects; + } + } + + public static void rename(@NotNull Container c, User user, String name) + { + rename(c, user, name, c.getTitle(), false); + } + + /** + * Transacted method to rename a container. Optionally, supports updating the title and aliasing the + * original container path when the name is changed (as name changes result in a new container path). + */ + public static Container rename(@NotNull Container c, User user, String name, @Nullable String title, boolean addAlias) + { + try (QuietCloser ignored = lockForMutation(MutatingOperation.rename, c); + DbScope.Transaction tx = ensureTransaction()) + { + final String oldName = c.getName(); + final String newName = StringUtils.trimToNull(name); + boolean isRenaming = !oldName.equals(newName); + StringBuilder errors = new StringBuilder(); + + // Rename + if (isRenaming) + { + // Issue 16221: Don't allow renaming of system reserved folders (e.g. /Shared, home, root, etc). + if (!isRenameable(c)) + throw new ApiUsageException("This folder may not be renamed as it is reserved by the system."); + + if (!Container.isLegalName(newName, c.isProject(), errors)) + throw new ApiUsageException(errors.toString()); + + // Issue 19061: Unable to do case-only container rename + if (c.getParent().hasChild(newName) && !c.equals(c.getParent().getChild(newName))) + { + if (c.getParent().isRoot()) + throw new ApiUsageException("The server already has a project with this name."); + throw new ApiUsageException("The " + (c.getParent().isProject() ? "project " : "folder ") + c.getParent().getPath() + " already has a folder with this name."); + } + + new SqlExecutor(CORE.getSchema()).execute("UPDATE " + CORE.getTableInfoContainers() + " SET Name=? WHERE EntityId=?", newName, c.getId()); + clearCache(); // Clear the entire cache, since containers cache their full paths + // Get new version since name has changed. + Container renamedContainer = getForId(c.getId()); + fireRenameContainer(renamedContainer, user, oldName); + // Clear again after the commit has propagated the state to other threads and transactions + // Do this in a commit task in case we've joined another existing DbScope.Transaction instead of starting our own + tx.addCommitTask(ContainerManager::clearCache, DbScope.CommitTaskOption.POSTCOMMIT); + + // Alias + if (addAlias) + { + // Intentionally use original container rather than the already renamedContainer + List newAliases = new ArrayList<>(getAliasesForContainer(c)); + newAliases.add(c.getPath()); + saveAliasesForContainer(c, newAliases, user); + } + } + + // Title + if (!c.getTitle().equals(title)) + { + if (!Container.isLegalTitle(title, errors)) + throw new ApiUsageException(errors.toString()); + updateTitle(c, title, user); + } + + tx.commit(); + } + catch (ValidationException e) + { + throw new IllegalArgumentException(e); + } + + return getForId(c.getId()); + } + + public static void setChildOrderToAlphabetical(Container parent) + { + setChildOrder(parent.getChildren(), true); + } + + public static void setChildOrder(Container parent, List orderedChildren) throws ContainerException + { + for (Container child : orderedChildren) + { + if (child == null || child.getParent() == null || !child.getParent().equals(parent)) // #13481 + throw new ContainerException("Invalid parent container of " + (child == null ? "null child container" : child.getPath())); + } + setChildOrder(orderedChildren, false); + } + + private static void setChildOrder(List siblings, boolean resetToAlphabetical) + { + try (DbScope.Transaction t = ensureTransaction()) + { + for (int index = 0; index < siblings.size(); index++) + { + Container current = siblings.get(index); + new SqlExecutor(CORE.getSchema()).execute("UPDATE " + CORE.getTableInfoContainers() + " SET SortOrder = ? WHERE EntityId = ?", + resetToAlphabetical ? 0 : index, current.getId()); + } + // Clear after the commit has propagated the state to other threads and transactions + // Do this in a commit task in case we've joined another existing DbScope.Transaction instead of starting our own + t.addCommitTask(ContainerManager::clearCache, DbScope.CommitTaskOption.POSTCOMMIT); + + t.commit(); + } + } + + private enum MutatingOperation + { + delete, + rename, + move + } + + private static final Map mutatingContainers = Collections.synchronizedMap(new IntHashMap<>()); + + private static QuietCloser lockForMutation(MutatingOperation op, Container c) + { + return lockForMutation(op, Collections.singletonList(c)); + } + + private static QuietCloser lockForMutation(MutatingOperation op, Collection containers) + { + List ids = new ArrayList<>(containers.size()); + synchronized (mutatingContainers) + { + for (Container container : containers) + { + MutatingOperation currentOp = mutatingContainers.get(container.getRowId()); + if (currentOp != null) + { + throw new ApiUsageException("Cannot start a " + op + " operation on " + container.getPath() + ". It is currently undergoing a " + currentOp); + } + ids.add(container.getRowId()); + } + ids.forEach(id -> mutatingContainers.put(id, op)); + } + return () -> + { + synchronized (mutatingContainers) + { + ids.forEach(mutatingContainers::remove); + } + }; + } + + // Delete containers from the database + private static boolean delete(final Collection containers, User user, @Nullable String comment) + { + // Do this check before we bother with any synchronization + for (Container container : containers) + { + if (!isDeletable(container)) + { + throw new ApiUsageException("Cannot delete container: " + container.getPath()); + } + } + + try (QuietCloser ignored = lockForMutation(MutatingOperation.delete, containers)) + { + boolean deleted = true; + for (Container c : containers) + { + deleted = deleted && delete(c, user, comment); + } + return deleted; + } + } + + // Delete a container from the database + private static boolean delete(final Container c, User user, @Nullable String comment) + { + // Verify method isn't called inappropriately + if (mutatingContainers.get(c.getRowId()) != MutatingOperation.delete) + { + throw new IllegalStateException("Container not flagged as being deleted: " + c.getPath()); + } + + LOG.debug("Starting container delete for " + c.getContainerNoun(true) + " " + c.getPath()); + + // Tell the search indexer to drop work for the container that's about to be deleted + SearchService.get().purgeForContainer(c); + + DbScope.RetryFn tryDeleteContainer = (tx) -> + { + // Verify that no children exist + Selector sel = new TableSelector(CORE.getTableInfoContainers(), new SimpleFilter(FieldKey.fromParts("Parent"), c), null); + + if (sel.exists()) + { + _removeFromCache(c, true); + return false; + } + + if (c.shouldRemoveFromPortal()) + { + // Need to remove portal page, too; container name is page's pageId and in container's parent container + Portal.PortalPage page = Portal.getPortalPage(c.getParent(), c.getName()); + if (null != page) // Be safe + Portal.deletePage(page); + + // Tell parent + setContainerTabDeleted(c.getParent(), c.getName(), c.getParent().getFolderType().getName()); + } + + fireDeleteContainer(c, user); + + SqlExecutor sqlExecutor = new SqlExecutor(CORE.getSchema()); + sqlExecutor.execute("DELETE FROM " + CORE.getTableInfoContainerAliases() + " WHERE ContainerRowId=?", c.getRowId()); + sqlExecutor.execute("DELETE FROM " + CORE.getTableInfoContainers() + " WHERE EntityId=?", c.getId()); + // now that the container is actually gone, delete all ACLs (better to have an ACL w/o object than object w/o ACL) + SecurityPolicyManager.removeAll(c); + // and delete all container-based sequences + DbSequenceManager.deleteAll(c); + + ExperimentService experimentService = ExperimentService.get(); + if (experimentService != null) + experimentService.removeContainerDataTypeExclusions(c.getId()); + + // After we've committed the transaction, be sure that we remove this container from the cache + // See https://www.labkey.org/issues/home/Developer/issues/details.view?issueId=17015 + tx.addCommitTask(() -> + { + // Be sure that we've waited until any threads that might be populating the cache have finished + // before we guarantee that we've removed this now-deleted container + DATABASE_QUERY_LOCK.lock(); + try + { + _removeFromCache(c, true); + } + finally + { + DATABASE_QUERY_LOCK.unlock(); + } + }, DbScope.CommitTaskOption.POSTCOMMIT); + String auditComment = c.getContainerNoun(true) + " " + c.getPath() + " was deleted"; + if (comment != null) + auditComment = auditComment.concat(". " + comment); + addAuditEvent(user, c, auditComment); + return true; + }; + + boolean success = CORE.getSchema().getScope().executeWithRetry(tryDeleteContainer); + if (success) + { + LOG.debug("Completed container delete for " + c.getContainerNoun(true) + " " + c.getPath()); + } + else + { + LOG.warn("Failed to delete container: " + c.getPath()); + } + return success; + } + + /** + * Delete a single container. Primarily for use by tests. + */ + public static boolean delete(final Container c, User user) + { + return delete(List.of(c), user, null); + } + + public static boolean isDeletable(Container c) + { + return !isSystemContainer(c); + } + + public static boolean isRenameable(Container c) + { + return !isSystemContainer(c); + } + + /** System containers include the root container, /Home, and /Shared */ + public static boolean isSystemContainer(Container c) + { + return c.equals(getRoot()) || c.equals(getHomeContainer()) || c.equals(getSharedContainer()); + } + + /** Has the container already been deleted or is it in the process of being deleted? */ + public static boolean exists(@Nullable Container c) + { + return c != null && null != getForId(c.getEntityId()) && mutatingContainers.get(c.getRowId()) != MutatingOperation.delete; + } + + public static void deleteAll(Container root, User user, @Nullable String comment) throws UnauthorizedException + { + if (!hasTreePermission(root, user, DeletePermission.class)) + throw new UnauthorizedException("You don't have delete permissions to all folders"); + + LOG.debug("Starting container (and children) delete for " + root.getContainerNoun(true) + " " + root.getPath()); + Set depthFirst = getAllChildrenDepthFirst(root); + depthFirst.add(root); + + delete(depthFirst, user, comment); + + LOG.debug("Completed container (and children) delete for " + root.getContainerNoun(true) + " " + root.getPath()); + } + + public static void deleteAll(Container root, User user) throws UnauthorizedException + { + deleteAll(root, user, null); + } + + private static void addAuditEvent(User user, Container c, String comment) + { + if (user != null) + { + AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, c, comment); + AuditLogService.get().addEvent(user, event); + } + } + + private static Set getAllChildrenDepthFirst(Container c) + { + Set set = new LinkedHashSet<>(); + getAllChildrenDepthFirst(c, set); + return set; + } + + private static void getAllChildrenDepthFirst(Container c, Collection list) + { + for (Container child : c.getChildren()) + { + getAllChildrenDepthFirst(child, list); + list.add(child); + } + } + + private static Container _getFromCachePath(Path path) + { + return CACHE_PATH.get(path); + } + + private static Container _addToCache(Container c) + { + assert DATABASE_QUERY_LOCK.isHeldByCurrentThread() : "Any cache modifications must be synchronized at a " + + "higher level so that we ensure that the container to be inserted still exists and hasn't been deleted"; + CACHE_ENTITY_ID.put(c.getEntityId(), c); + CACHE_PATH.put(c.getParsedPath(), c); + return c; + } + + private static void _clearChildrenFromCache(Container c) + { + CACHE_CHILDREN.remove(c.getEntityId()); + navTreeManageUncache(c); + } + + /** @param hierarchyChange whether the shape of the container tree has changed */ + private static void _removeFromCache(Container c, boolean hierarchyChange) + { + CACHE_ENTITY_ID.remove(c.getEntityId()); + CACHE_PATH.remove(c.getParsedPath()); + + if (hierarchyChange) + { + // This is strictly keeping track of the parent/child relationships themselves so it only needs to be + // cleared when the tree changes + CACHE_CHILDREN.clear(); + } + + navTreeManageUncache(c); + } + + public static void clearCache() + { + CACHE_PATH.clear(); + CACHE_ENTITY_ID.clear(); + CACHE_CHILDREN.clear(); + + // UNDONE: NavTreeManager should register a ContainerListener + NavTreeManager.uncacheAll(); + } + + private static void navTreeManageUncache(Container c) + { + // UNDONE: NavTreeManager should register a ContainerListener + NavTreeManager.uncacheTree(PROJECT_LIST_ID); + NavTreeManager.uncacheTree(getRoot().getId()); + + Container project = c.getProject(); + if (project != null) + { + NavTreeManager.uncacheTree(project.getId()); + NavTreeManager.uncacheTree(PROJECT_LIST_ID + project.getId()); + } + } + + public static void notifyContainerChange(String id, Property prop) + { + notifyContainerChange(id, prop, null); + } + + public static void notifyContainerChange(String id, Property prop, @Nullable User u) + { + if (_constructing.contains(new GUID(id))) + return; + + Container c = getForId(id); + if (null != c) + { + _removeFromCache(c, false); + c = getForId(id); // load a fresh container since the original might be stale. + if (null != c) + { + ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(c, u, prop, null, null); + firePropertyChangeEvent(evt); + } + } + } + + + /** Recursive, including root node */ + public static Set getAllChildren(Container root) + { + Set children = getAllChildrenDepthFirst(root); + children.add(root); + + return Collections.unmodifiableSet(children); + } + + /** + * Return all children of the root node, including root node, which have the given active module + */ + @NotNull + public static Set getAllChildrenWithModule(@NotNull Container root, @NotNull Module module) + { + Set children = new HashSet<>(); + for (Container candidate : getAllChildren(root)) + { + if (candidate.getActiveModules().contains(module)) + children.add(candidate); + } + return Collections.unmodifiableSet(children); + } + + public static long getContainerCount() + { + return new TableSelector(CORE.getTableInfoContainers()).getRowCount(); + } + + public static long getWorkbookCount() + { + return new TableSelector(CORE.getTableInfoContainers(), new SimpleFilter(FieldKey.fromParts("type"), "workbook"), null).getRowCount(); + } + + public static long getArchivedContainerCount() + { + return new TableSelector(CORE.getTableInfoContainers(), new SimpleFilter(FieldKey.fromParts("lockstate"), "Archived"), null).getRowCount(); + } + + public static long getAuditCommentRequiredCount() + { + SQLFragment sql = new SQLFragment( + "SELECT COUNT(*) FROM\n" + + " core.containers c\n" + + " JOIN prop.propertysets ps on c.entityid = ps.objectid\n" + + " JOIN prop.properties p on p.\"set\" = ps.\"set\"\n" + + "WHERE ps.category = '" + AUDIT_SETTINGS_PROPERTY_SET_NAME + "' AND p.name='"+ REQUIRE_USER_COMMENTS_PROPERTY_NAME + "' and p.value='true'"); + return new SqlSelector(CORE.getSchema(), sql).getObject(Long.class); + } + + + /** Retrieve entire container hierarchy */ + public static MultiValuedMap getContainerTree() + { + final MultiValuedMap mm = new ArrayListValuedHashMap<>(); + + // Get all containers and parents + SqlSelector selector = new SqlSelector(CORE.getSchema(), "SELECT Parent, EntityId FROM " + CORE.getTableInfoContainers() + " ORDER BY SortOrder, LOWER(Name) ASC"); + + selector.forEach(rs -> { + String parentId = rs.getString(1); + Container parent = (parentId != null ? getForId(parentId) : null); + Container child = getForId(rs.getString(2)); + + if (null != child) + mm.put(parent, child); + }); + + return mm; + } + + /** + * Returns a branch of the container tree including only the root and its descendants + * @param root The root container + * @return MultiMap of containers including root and its descendants + */ + public static MultiValuedMap getContainerTree(Container root) + { + //build a multimap of only the container ids + final MultiValuedMap mmIds = new ArrayListValuedHashMap<>(); + + // Get all containers and parents + Selector selector = new SqlSelector(CORE.getSchema(), "SELECT Parent, EntityId FROM " + CORE.getTableInfoContainers() + " ORDER BY SortOrder, LOWER(Name) ASC"); + + selector.forEach(rs -> mmIds.put(rs.getString(1), rs.getString(2))); + + //now find the root and build a MultiMap of it and its descendants + MultiValuedMap mm = new ArrayListValuedHashMap<>(); + mm.put(null, root); + addChildren(root, mmIds, mm); + return mm; + } + + private static void addChildren(Container c, MultiValuedMap mmIds, MultiValuedMap mm) + { + Collection childIds = mmIds.get(c.getId()); + if (null != childIds) + { + for (String childId : childIds) + { + Container child = getForId(childId); + if (null != child) + { + mm.put(c, child); + addChildren(child, mmIds, mm); + } + } + } + } + + public static Set getContainerSet(MultiValuedMap mm, User user, Class perm) + { + Collection containers = mm.values(); + if (null == containers) + return new HashSet<>(); + + return containers + .stream() + .filter(c -> c.hasPermission(user, perm)) + .collect(Collectors.toSet()); + } + + + public static SQLFragment getIdsAsCsvList(Set containers, SqlDialect d) + { + if (containers.isEmpty()) + return new SQLFragment("(NULL)"); // WHERE x IN (NULL) should match no rows + + SQLFragment csvList = new SQLFragment("("); + String comma = ""; + for (Container container : containers) + { + csvList.append(comma); + comma = ","; + csvList.appendValue(container, d); + } + csvList.append(")"); + + return csvList; + } + + + public static List getIds(User user, Class perm) + { + Set containers = getContainerSet(getContainerTree(), user, perm); + + List ids = new ArrayList<>(containers.size()); + + for (Container c : containers) + ids.add(c.getId()); + + return ids; + } + + + // + // ContainerListener + // + + public interface ContainerListener extends PropertyChangeListener + { + enum Order {First, Last} + + /** Called after a new container has been created */ + void containerCreated(Container c, User user); + + default void containerCreated(Container c, User user, @Nullable String auditMsg) + { + containerCreated(c, user); + } + + /** Called immediately prior to deleting the row from core.containers */ + void containerDeleted(Container c, User user); + + /** Called after the container has been moved to its new parent */ + void containerMoved(Container c, Container oldParent, User user); + + /** + * Called prior to moving a container, to find out if there are any issues that would prevent a successful move + * @return a list of errors that should prevent the move from happening, if any + */ + @NotNull + Collection canMove(Container c, Container newParent, User user); + + @Override + void propertyChange(PropertyChangeEvent evt); + } + + public static abstract class AbstractContainerListener implements ContainerListener + { + @Override + public void containerCreated(Container c, User user) + {} + + @Override + public void containerDeleted(Container c, User user) + {} + + @Override + public void containerMoved(Container c, Container oldParent, User user) + {} + + @NotNull + @Override + public Collection canMove(Container c, Container newParent, User user) + { + return Collections.emptyList(); + } + + @Override + public void propertyChange(PropertyChangeEvent evt) + {} + } + + + public static class ContainerPropertyChangeEvent extends PropertyChangeEvent implements PropertyChange + { + public final Property property; + public final Container container; + public User user; + + public ContainerPropertyChangeEvent(Container c, @Nullable User user, Property p, Object oldValue, Object newValue) + { + super(c, p.name(), oldValue, newValue); + container = c; + this.user = user; + property = p; + } + + public ContainerPropertyChangeEvent(Container c, Property p, Object oldValue, Object newValue) + { + this(c, null, p, oldValue, newValue); + } + + @Override + public Property getProperty() + { + return property; + } + } + + + // Thread-safe list implementation that allows iteration and modifications without external synchronization + private static final List _listeners = new CopyOnWriteArrayList<>(); + private static final List _laterListeners = new CopyOnWriteArrayList<>(); + + // These listeners are executed in the order they are registered, before the "Last" listeners + public static void addContainerListener(ContainerListener listener) + { + addContainerListener(listener, ContainerListener.Order.First); + } + + + // Explicitly request "Last" ordering via this method. "Last" listeners execute after all "First" listeners. + public static void addContainerListener(ContainerListener listener, ContainerListener.Order order) + { + if (ContainerListener.Order.First == order) + _listeners.add(listener); + else + _laterListeners.add(listener); + } + + + public static void removeContainerListener(ContainerListener listener) + { + _listeners.remove(listener); + _laterListeners.remove(listener); + } + + + private static List getListeners() + { + List combined = new ArrayList<>(_listeners.size() + _laterListeners.size()); + combined.addAll(_listeners); + combined.addAll(_laterListeners); + + return combined; + } + + + private static List getListenersReversed() + { + List combined = new LinkedList<>(); + + // Copy to guarantee consistency between .listIterator() and .size() + List copy = new ArrayList<>(_listeners); + ListIterator iter = copy.listIterator(copy.size()); + + // Iterate in reverse + while(iter.hasPrevious()) + combined.add(iter.previous()); + + // Copy to guarantee consistency between .listIterator() and .size() + // Add elements from the laterList in reverse order so that Core is fired last + List laterCopy = new ArrayList<>(_laterListeners); + ListIterator laterIter = laterCopy.listIterator(laterCopy.size()); + + // Iterate in reverse + while(laterIter.hasPrevious()) + combined.add(laterIter.previous()); + + return combined; + } + + + protected static void fireCreateContainer(Container c, User user, @Nullable String auditMsg) + { + List list = getListeners(); + + for (ContainerListener cl : list) + { + try + { + cl.containerCreated(c, user, auditMsg); + } + catch (Throwable t) + { + LOG.error("fireCreateContainer for " + cl.getClass().getName(), t); + } + } + } + + + protected static void fireDeleteContainer(Container c, User user) + { + List list = getListenersReversed(); + + for (ContainerListener l : list) + { + LOG.debug("Deleting " + c.getPath() + ": fireDeleteContainer for " + l.getClass().getName()); + try + { + l.containerDeleted(c, user); + } + catch (RuntimeException e) + { + LOG.error("fireDeleteContainer for " + l.getClass().getName(), e); + + // Fail fast (first Throwable aborts iteration), #17560 + throw e; + } + } + } + + + protected static void fireRenameContainer(Container c, User user, String oldValue) + { + ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(c, user, Property.Name, oldValue, c.getName()); + firePropertyChangeEvent(evt); + } + + + protected static void fireMoveContainer(Container c, Container oldParent, User user) + { + List list = getListeners(); + + for (ContainerListener cl : list) + { + // While we would ideally transact the full container move, that will likely cause long-blocking + // queries and/or deadlocks. For now, at least transact each separate move handler independently + try (DbScope.Transaction transaction = CoreSchema.getInstance().getSchema().getScope().ensureTransaction()) + { + cl.containerMoved(c, oldParent, user); + transaction.commit(); + } + } + ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(c, user, Property.Parent, oldParent, c.getParent()); + firePropertyChangeEvent(evt); + } + + + public static void firePropertyChangeEvent(ContainerPropertyChangeEvent evt) + { + if (_constructing.contains(evt.container.getEntityId())) + return; + + List list = getListeners(); + for (ContainerListener l : list) + { + try + { + l.propertyChange(evt); + } + catch (Throwable t) + { + LOG.error("firePropertyChangeEvent for " + l.getClass().getName(), t); + } + } + } + + private static final List MODULE_DEPENDENCY_PROVIDERS = new CopyOnWriteArrayList<>(); + + public static void registerModuleDependencyProvider(ModuleDependencyProvider provider) + { + MODULE_DEPENDENCY_PROVIDERS.add(provider); + } + + public static void forEachModuleDependencyProvider(Consumer action) + { + MODULE_DEPENDENCY_PROVIDERS.forEach(action); + } + + // Compliance module adds a locked project handler that checks permissions; without that, this implementation + // is used, and projects are never locked + static volatile LockedProjectHandler LOCKED_PROJECT_HANDLER = (project, user, contextualRoles, lockState) -> false; + + // Replaces any previously set LockedProjectHandler + public static void setLockedProjectHandler(LockedProjectHandler handler) + { + LOCKED_PROJECT_HANDLER = handler; + } + + public static Container createDefaultSupportContainer() + { + LOG.info("Creating default support container: " + DEFAULT_SUPPORT_PROJECT_PATH); + // create a "support" container. Admins can do anything, + // Users can read/write, Guests can read. + return bootstrapContainer(DEFAULT_SUPPORT_PROJECT_PATH, + RoleManager.getRole(AuthorRole.class), + RoleManager.getRole(ReaderRole.class) + ); + } + + public static void removeDefaultSupportContainer(User user) + { + Container support = getDefaultSupportContainer(); + if (support != null) + { + LOG.info("Removing default support container: " + DEFAULT_SUPPORT_PROJECT_PATH); + ContainerManager.delete(support, user); + } + } + + public static Container getDefaultSupportContainer() + { + return getForPath(DEFAULT_SUPPORT_PROJECT_PATH); + } + + public static List getAliasesForContainer(Container c) + { + return Collections.unmodifiableList(new SqlSelector(CORE.getSchema(), + new SQLFragment("SELECT Path FROM " + CORE.getTableInfoContainerAliases() + " WHERE ContainerRowId = ? ORDER BY Path", + c.getRowId())).getArrayList(String.class)); + } + + @Nullable + public static Container resolveContainerPathAlias(String path) + { + return resolveContainerPathAlias(path, false); + } + + @Nullable + private static Container resolveContainerPathAlias(String path, boolean top) + { + // Strip any trailing slashes + while (path.endsWith("/")) + { + path = path.substring(0, path.length() - 1); + } + + // Simple case -- resolve directly (sans alias) + Container aliased = getForPath(path); + if (aliased != null) + return aliased; + + // Simple case -- directly resolve from database + aliased = getForPathAlias(path); + if (aliased != null) + return aliased; + + // At the leaf and the container was not found + if (top) + return null; + + List splits = Arrays.asList(path.split("/")); + String subPath = ""; + for (int i=0; i < splits.size()-1; i++) // minus 1 due to leaving off last container + { + if (!splits.get(i).isEmpty()) + subPath += "/" + splits.get(i); + } + + aliased = resolveContainerPathAlias(subPath, false); + + if (aliased == null) + return null; + + String leafPath = aliased.getPath() + "/" + splits.get(splits.size()-1); + return resolveContainerPathAlias(leafPath, true); + } + + @Nullable + private static Container getForPathAlias(String path) + { + // We store the path as lower-case, so we don't need to also LOWER() on the value in core.ContainerAliases, letting the DB use the index + Container[] ret = new SqlSelector(CORE.getSchema(), + "SELECT * FROM " + CORE.getTableInfoContainers() + " c, " + CORE.getTableInfoContainerAliases() + " ca WHERE ca.ContainerRowId = c.RowId AND ca.path = LOWER(?)", + path).getArray(Container.class); + + return ret.length == 0 ? null : ret[0]; + } + + public static Container getMoveTargetContainer(@Nullable String queryName, @NotNull Container sourceContainer, User user, @Nullable String targetIdOrPath, Errors errors) + { + if (targetIdOrPath == null) + { + errors.reject(ERROR_GENERIC, "A target container must be specified for the move operation."); + return null; + } + + Container _targetContainer = getContainerForIdOrPath(targetIdOrPath); + if (_targetContainer == null) + { + errors.reject(ERROR_GENERIC, "The target container was not found: " + targetIdOrPath + "."); + return null; + } + + if (!_targetContainer.hasPermission(user, InsertPermission.class)) + { + String _queryName = queryName == null ? "this table" : "'" + queryName + "'"; + errors.reject(ERROR_GENERIC, "You do not have permission to move rows from " + _queryName + " to the target container: " + targetIdOrPath + "."); + return null; + } + + if (!isValidTargetContainer(sourceContainer, _targetContainer)) + { + errors.reject(ERROR_GENERIC, "Invalid target container for the move operation: " + targetIdOrPath + "."); + return null; + } + return _targetContainer; + } + + private static Container getContainerForIdOrPath(String targetContainer) + { + Container c = ContainerManager.getForId(targetContainer); + if (c == null) + c = ContainerManager.getForPath(targetContainer); + + return c; + } + + // targetContainer must be in the same app project at this time + // i.e. child of current project, project of current child, sibling within project + private static boolean isValidTargetContainer(Container current, Container target) + { + if (current.isRoot() || target.isRoot()) + return false; + + // Allow moving to the current container since we now allow the chosen entities to be from different containers + if (current.equals(target)) + return true; + + boolean moveFromProjectToChild = current.isProject() && target.getParent().equals(current); + boolean moveFromChildToProject = !current.isProject() && current.getParent().isProject() && current.getParent().equals(target); + boolean moveFromChildToSibling = !current.isProject() && current.getParent().isProject() && current.getParent().equals(target.getParent()); + + return moveFromProjectToChild || moveFromChildToProject || moveFromChildToSibling; + } + + /** + * Updates the container of specified rows in the provided database table. Optionally, the modification timestamp + * and the user who made the modification can also be updated if specified. + * + * @param dataTable The table where the container update should be applied. + * @param idField The name of the identifier field used to locate the rows to update. + * @param ids A collection of identifier values specifying the rows to be updated. + * @param targetContainer The target container to set for the specified rows. + * @param user The user performing the update. If null, modified/modifiedBy details are not updated. + * @param withModified If true, updates the modified timestamp and the user who made the modification. + * @return The number of rows updated in the table. + */ + public static int updateContainer(TableInfo dataTable, String idField, Collection ids, Container targetContainer, @Nullable User user, boolean withModified) + { + try (DbScope.Transaction transaction = dataTable.getSchema().getScope().ensureTransaction()) + { + SQLFragment dataUpdate = new SQLFragment("UPDATE ").append(dataTable) + .append(" SET container = ").appendValue(targetContainer); + if (withModified) + { + assert user != null : "User must be specified when updating modified/modifiedBy details."; + dataUpdate.append(", modified = ").appendValue(new Date()); + dataUpdate.append(", modifiedby = ").appendValue(user.getUserId()); + } + dataUpdate.append(" WHERE ").appendIdentifier(idField); + dataTable.getSchema().getSqlDialect().appendInClauseSql(dataUpdate, ids); + int numUpdated = new SqlExecutor(dataTable.getSchema()).execute(dataUpdate); + transaction.commit(); + + return numUpdated; + } + } + + /** + * If a container at the given path does not exist, create one and set permissions. If the container does exist, + * permissions are only set if there is no explicit ACL for the container. This prevents us from resetting + * permissions if all users are dropped. Implicitly done as an admin-level service user. + */ + @NotNull + public static Container bootstrapContainer(String path, @NotNull Role userRole, @Nullable Role guestRole) + { + Container c = null; + User user = User.getAdminServiceUser(); + + try + { + c = getForPath(path); + } + catch (RootContainerException e) + { + // Ignore this -- root doesn't exist yet + } + boolean newContainer = false; + + if (c == null) + { + LOG.debug("Creating new container for path '" + path + "'"); + newContainer = true; + c = ensureContainer(path, user); + } + + // Only set permissions if there are no explicit permissions + // set for this object or we just created it + Integer policyCount = null; + if (!newContainer) + { + policyCount = new SqlSelector(CORE.getSchema(), + "SELECT COUNT(*) FROM " + CORE.getTableInfoPolicies() + " WHERE ResourceId = ?", + c.getId()).getObject(Integer.class); + } + + if (newContainer || 0 == policyCount.intValue()) + { + LOG.debug("Setting permissions for '" + path + "'"); + MutableSecurityPolicy policy = new MutableSecurityPolicy(c); + policy.addRoleAssignment(SecurityManager.getGroup(Group.groupUsers), userRole); + if (guestRole != null) + policy.addRoleAssignment(SecurityManager.getGroup(Group.groupGuests), guestRole); + SecurityPolicyManager.savePolicy(policy, user); + } + + return c; + } + + /** + * @param container the container being created. May be null if we haven't actually created it yet + * @param parent the parent of the container being created. Used in case the container doesn't actually exist yet. + * @return the list of standard steps and any extra ones based on the container's FolderType + */ + public static List getCreateContainerWizardSteps(@Nullable Container container, @NotNull Container parent) + { + List navTrail = new ArrayList<>(); + + boolean isProject = parent.isRoot(); + + navTrail.add(new NavTree(isProject ? "Create Project" : "Create Folder")); + navTrail.add(new NavTree("Users / Permissions")); + if (isProject) + navTrail.add(new NavTree("Project Settings")); + if (container != null) + navTrail.addAll(container.getFolderType().getExtraSetupSteps(container)); + return navTrail; + } + + @TestTimeout(120) @TestWhen(TestWhen.When.BVT) + public static class TestCase extends Assert implements ContainerListener + { + Map _containers = new HashMap<>(); + Container _testRoot = null; + + @Before + public void setUp() + { + if (null == _testRoot) + { + Container junit = JunitUtil.getTestContainer(); + _testRoot = ensureContainer(junit, "ContainerManager$TestCase-" + GUID.makeGUID(), TestContext.get().getUser()); + addContainerListener(this); + } + } + + @After + public void tearDown() + { + removeContainerListener(this); + if (null != _testRoot) + deleteAll(_testRoot, TestContext.get().getUser()); + } + + @Test + public void testImproperFolderNamesBlocked() + { + String[] badNames = {"", "f\\o", "f/o", "f\\\\o", "foo;", "@foo", "foo" + '\u001F', '\u0000' + "foo", "fo" + '\u007F' + "o", "" + '\u009F'}; + + for (String name: badNames) + { + try + { + Container c = createContainer(_testRoot, name, TestContext.get().getUser()); + try + { + assertTrue(delete(c, TestContext.get().getUser())); + } + catch (Exception ignored) {} + fail("Should have thrown exception when trying to create container with name: " + name); + } + catch (ApiUsageException e) + { + // Do nothing, this is expected + } + } + } + + @Test + public void testCreateDeleteContainers() + { + int count = 20; + Random random = new Random(); + MultiValuedMap mm = new ArrayListValuedHashMap<>(); + + for (int i = 1; i <= count; i++) + { + int parentId = random.nextInt(i); + String parentName = 0 == parentId ? _testRoot.getName() : String.valueOf(parentId); + String childName = String.valueOf(i); + mm.put(parentName, childName); + } + + logNode(mm, _testRoot.getName(), 0); + for (int i=0; i<2; i++) //do this twice to make sure the containers were *really* deleted + { + createContainers(mm, _testRoot.getName(), _testRoot); + assertEquals(count, _containers.size()); + cleanUpChildren(mm, _testRoot.getName(), _testRoot); + assertEquals(0, _containers.size()); + } + } + + @Test + public void testCache() + { + assertEquals(0, _containers.size()); + assertEquals(0, getChildren(_testRoot).size()); + + Container one = createContainer(_testRoot, "one", TestContext.get().getUser()); + assertEquals(1, _containers.size()); + assertEquals(1, getChildren(_testRoot).size()); + assertEquals(0, getChildren(one).size()); + + Container oneA = createContainer(one, "A", TestContext.get().getUser()); + assertEquals(2, _containers.size()); + assertEquals(1, getChildren(_testRoot).size()); + assertEquals(1, getChildren(one).size()); + assertEquals(0, getChildren(oneA).size()); + + Container oneB = createContainer(one, "B", TestContext.get().getUser()); + assertEquals(3, _containers.size()); + assertEquals(1, getChildren(_testRoot).size()); + assertEquals(2, getChildren(one).size()); + assertEquals(0, getChildren(oneB).size()); + + Container deleteme = createContainer(one, "deleteme", TestContext.get().getUser()); + assertEquals(4, _containers.size()); + assertEquals(1, getChildren(_testRoot).size()); + assertEquals(3, getChildren(one).size()); + assertEquals(0, getChildren(deleteme).size()); + + assertTrue(delete(deleteme, TestContext.get().getUser())); + assertEquals(3, _containers.size()); + assertEquals(1, getChildren(_testRoot).size()); + assertEquals(2, getChildren(one).size()); + + Container oneC = createContainer(one, "C", TestContext.get().getUser()); + assertEquals(4, _containers.size()); + assertEquals(1, getChildren(_testRoot).size()); + assertEquals(3, getChildren(one).size()); + assertEquals(0, getChildren(oneC).size()); + + assertTrue(delete(oneC, TestContext.get().getUser())); + assertTrue(delete(oneB, TestContext.get().getUser())); + assertEquals(1, getChildren(one).size()); + + assertTrue(delete(oneA, TestContext.get().getUser())); + assertEquals(0, getChildren(one).size()); + + assertTrue(delete(one, TestContext.get().getUser())); + assertEquals(0, getChildren(_testRoot).size()); + assertEquals(0, _containers.size()); + } + + @Test + public void testFolderType() + { + // Test all folder types + List folderTypes = new ArrayList<>(FolderTypeManager.get().getAllFolderTypes()); + for (FolderType folderType : folderTypes) + { + if (!folderType.isProjectOnlyType()) // Dataspace can't be subfolder + testOneFolderType(folderType); + } + } + + private void testOneFolderType(FolderType folderType) + { + LOG.info("testOneFolderType(" + folderType.getName() + "): creating container"); + Container newFolder = createContainer(_testRoot, "folderTypeTest", TestContext.get().getUser()); + FolderType ft = newFolder.getFolderType(); + assertEquals(FolderType.NONE, ft); + + Container newFolderFromCache = getForId(newFolder.getId()); + assertNotNull(newFolderFromCache); + assertEquals(FolderType.NONE, newFolderFromCache.getFolderType()); + LOG.info("testOneFolderType(" + folderType.getName() + "): setting folder type"); + newFolder.setFolderType(folderType, TestContext.get().getUser()); + + newFolderFromCache = getForId(newFolder.getId()); + assertNotNull(newFolderFromCache); + assertEquals(newFolderFromCache.getFolderType().getName(), folderType.getName()); + assertEquals(newFolderFromCache.getFolderType().getDescription(), folderType.getDescription()); + + LOG.info("testOneFolderType(" + folderType.getName() + "): deleteAll"); + deleteAll(newFolder, TestContext.get().getUser()); // There might be subfolders because of container tabs + LOG.info("testOneFolderType(" + folderType.getName() + "): deleteAll complete"); + Container deletedContainer = getForId(newFolder.getId()); + + if (deletedContainer != null) + { + fail("Expected container with Id " + newFolder.getId() + " to be deleted, but found " + deletedContainer + ". Folder type was " + folderType); + } + } + + private static void createContainers(MultiValuedMap mm, String name, Container parent) + { + Collection nodes = mm.get(name); + + if (null == nodes) + return; + + for (String childName : nodes) + { + Container child = createContainer(parent, childName, TestContext.get().getUser()); + createContainers(mm, childName, child); + } + } + + private static void cleanUpChildren(MultiValuedMap mm, String name, Container parent) + { + Collection nodes = mm.get(name); + + if (null == nodes) + return; + + for (String childName : nodes) + { + Container child = getForPath(makePath(parent, childName)); + cleanUpChildren(mm, childName, child); + assertTrue(delete(child, TestContext.get().getUser())); + } + } + + private static void logNode(MultiValuedMap mm, String name, int offset) + { + Collection nodes = mm.get(name); + + if (null == nodes) + return; + + for (String childName : nodes) + { + LOG.debug(StringUtils.repeat(" ", offset) + childName); + logNode(mm, childName, offset + 1); + } + } + + // ContainerListener + @Override + public void propertyChange(PropertyChangeEvent evt) + { + } + + @Override + public void containerCreated(Container c, User user) + { + if (null == _testRoot || !c.getParsedPath().startsWith(_testRoot.getParsedPath())) + return; + _containers.put(c.getParsedPath(), c); + } + + + @Override + public void containerDeleted(Container c, User user) + { + _containers.remove(c.getParsedPath()); + } + + @Override + public void containerMoved(Container c, Container oldParent, User user) + { + } + + @NotNull + @Override + public Collection canMove(Container c, Container newParent, User user) + { + return Collections.emptyList(); + } + } + + static + { + ObjectFactory.Registry.register(Container.class, new ContainerFactory()); + } + + public static class ContainerFactory implements ObjectFactory + { + @Override + public Container fromMap(Map m) + { + throw new UnsupportedOperationException(); + } + + @Override + public Container fromMap(Container bean, Map m) + { + throw new UnsupportedOperationException(); + } + + @Override + public Map toMap(Container bean, Map m) + { + throw new UnsupportedOperationException(); + } + + @Override + public Container handle(ResultSet rs) throws SQLException + { + String id; + Container d; + String parentId = rs.getString("Parent"); + String name = rs.getString("Name"); + id = rs.getString("EntityId"); + int rowId = rs.getInt("RowId"); + int sortOrder = rs.getInt("SortOrder"); + Date created = rs.getTimestamp("Created"); + int createdBy = rs.getInt("CreatedBy"); + // _ts + String description = rs.getString("Description"); + String type = rs.getString("Type"); + String title = rs.getString("Title"); + boolean searchable = rs.getBoolean("Searchable"); + String lockStateString = rs.getString("LockState"); + LockState lockState = null != lockStateString ? Enums.getIfPresent(LockState.class, lockStateString).or(LockState.Unlocked) : LockState.Unlocked; + + LocalDate expirationDate = rs.getObject("ExpirationDate", LocalDate.class); + + // Could be running upgrade code before these recent columns have been added to the table. Use a find map + // to determine if they are present. Issue 51692. These checks could be removed after creation of these + // columns is incorporated into the bootstrap scripts. + Map findMap = ResultSetUtil.getFindMap(rs.getMetaData()); + Long fileRootSize = findMap.containsKey("FileRootSize") ? (Long)rs.getObject("FileRootSize") : null; // getObject() and cast because getLong() returns 0 for null + LocalDateTime fileRootLastCrawled = findMap.containsKey("FileRootLastCrawled") ? rs.getObject("FileRootLastCrawled", LocalDateTime.class) : null; + + Container dirParent = null; + if (null != parentId) + dirParent = getForId(parentId); + + d = new Container(dirParent, name, id, rowId, sortOrder, created, createdBy, searchable); + d.setDescription(description); + d.setType(type); + d.setTitle(title); + d.setLockState(lockState); + d.setExpirationDate(expirationDate); + d.setFileRootSize(fileRootSize); + d.setFileRootLastCrawled(fileRootLastCrawled); + return d; + } + + @Override + public ArrayList handleArrayList(ResultSet rs) throws SQLException + { + ArrayList list = new ArrayList<>(); + while (rs.next()) + { + list.add(handle(rs)); + } + return list; + } + } + + public static Container createFakeContainer(@Nullable String name, @Nullable Container parent) + { + return new Container(parent, name, GUID.makeGUID(), 1, 0, new Date(), 0, false); + } +} diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java index 097331515b0..13257230cfa 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java @@ -1925,7 +1925,7 @@ public Map moveSamples(Collection sample // move the events associated with the samples that have moved SampleTimelineAuditProvider auditProvider = new SampleTimelineAuditProvider(); int auditEventCount = auditProvider.moveEvents(targetContainer, sampleIds); - updateCounts.compute("sampleAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount ); + updateCounts.compute("sampleAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount); AuditBehaviorType stAuditBehavior = samplesTable.getEffectiveAuditBehavior(auditBehavior); // create new events for each sample that was moved. diff --git a/list/src/org/labkey/list/model/ListQueryUpdateService.java b/list/src/org/labkey/list/model/ListQueryUpdateService.java index 83b91f91a88..044999a80e8 100644 --- a/list/src/org/labkey/list/model/ListQueryUpdateService.java +++ b/list/src/org/labkey/list/model/ListQueryUpdateService.java @@ -1,841 +1,844 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.list.model; - -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.announcements.DiscussionService; -import org.labkey.api.attachments.AttachmentFile; -import org.labkey.api.attachments.AttachmentParent; -import org.labkey.api.attachments.AttachmentParentFactory; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.LookupResolutionType; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.Selector.ForEachBatchBlock; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.dataiterator.DataIteratorContext; -import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; -import org.labkey.api.exp.ObjectProperty; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.list.ListDefinition; -import org.labkey.api.exp.list.ListImportProgress; -import org.labkey.api.exp.list.ListItem; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.IPropertyValidator; -import org.labkey.api.exp.property.ValidatorContext; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.lists.permissions.ManagePicklistsPermission; -import org.labkey.api.query.AbstractQueryUpdateService; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.DefaultQueryUpdateService; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.InvalidKeyException; -import org.labkey.api.query.PropertyValidationError; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QueryUpdateServiceException; -import org.labkey.api.query.ValidationError; -import org.labkey.api.query.ValidationException; -import org.labkey.api.reader.DataLoader; -import org.labkey.api.security.ElevatedUser; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.MoveEntitiesPermission; -import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.security.roles.EditorRole; -import org.labkey.api.usageMetrics.SimpleMetricsService; -import org.labkey.api.util.GUID; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.UnexpectedException; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.writer.VirtualFile; -import org.labkey.list.view.ListItemAttachmentParent; - -import java.io.IOException; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.labkey.api.util.IntegerUtils.isIntegral; - -/** - * Implementation of QueryUpdateService for Lists - */ -public class ListQueryUpdateService extends DefaultQueryUpdateService -{ - private final ListDefinitionImpl _list; - private static final String ID = "entityId"; - - public ListQueryUpdateService(ListTable queryTable, TableInfo dbTable, @NotNull ListDefinition list) - { - super(queryTable, dbTable, createMVMapping(queryTable.getList().getDomain())); - _list = (ListDefinitionImpl) list; - } - - @Override - public void configureDataIteratorContext(DataIteratorContext context) - { - if (context.getInsertOption().batch) - { - context.setMaxRowErrors(100); - context.setFailFast(false); - } - - context.putConfigParameter(ConfigParameters.TrimStringRight, Boolean.TRUE); - } - - @Override - protected @Nullable AttachmentParentFactory getAttachmentParentFactory() - { - return new ListItemAttachmentParentFactory(); - } - - @Override - protected Map getRow(User user, Container container, Map listRow) throws InvalidKeyException - { - Map ret = null; - - if (null != listRow) - { - SimpleFilter keyFilter = getKeyFilter(listRow); - - if (null != keyFilter) - { - TableInfo queryTable = getQueryTable(); - Map raw = new TableSelector(queryTable, keyFilter, null).getMap(); - - if (null != raw && !raw.isEmpty()) - { - ret = new CaseInsensitiveHashMap<>(); - - // EntityId - ret.put("EntityId", raw.get("entityid")); - - for (DomainProperty prop : _list.getDomainOrThrow().getProperties()) - { - String propName = prop.getName(); - ColumnInfo column = queryTable.getColumn(propName); - Object value = column.getValue(raw); - if (value != null) - ret.put(propName, value); - } - } - } - } - - return ret; - } - - - @Override - public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, - @Nullable Map configParameters, Map extraScriptContext) - { - for (Map row : rows) - { - aliasColumns(getColumnMapping(), row); - } - - DataIteratorContext context = getDataIteratorContext(errors, InsertOption.INSERT, configParameters); - List> result = this._insertRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); - - if (null != result) - { - ListManager mgr = ListManager.get(); - - for (Map row : result) - { - if (null != row.get(ID)) - { - // Audit each row - String entityId = (String) row.get(ID); - String newRecord = mgr.formatAuditItem(_list, user, row); - - mgr.addAuditEvent(_list, user, container, "A new list record was inserted", entityId, null, newRecord); - } - } - - if (!result.isEmpty() && !errors.hasErrors()) - mgr.indexList(_list); - } - - return result; - } - - private User getListUser(User user, Container container) - { - if (_list.isPicklist() && container.hasPermission(user, ManagePicklistsPermission.class)) - { - // if the list is a picklist and you have permission to manage picklists, that equates - // to having editor permission. - return ElevatedUser.ensureContextualRoles(container, user, Pair.of(DeletePermission.class, EditorRole.class)); - } - return user; - } - - @Override - protected @Nullable List> _insertRowsUsingDIB(User user, Container container, List> rows, DataIteratorContext context, @Nullable Map extraScriptContext) - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to insert data into this table."); - - return super._insertRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); - } - - @Override - protected @Nullable List> _updateRowsUsingDIB(User user, Container container, List> rows, - DataIteratorContext context, @Nullable Map extraScriptContext) - { - if (!hasPermission(user, UpdatePermission.class)) - throw new UnauthorizedException("You do not have permission to update data in this table."); - - return super._updateRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); - } - - public int insertUsingDataIterator(DataLoader loader, User user, Container container, BatchValidationException errors, @Nullable VirtualFile attachmentDir, - @Nullable ListImportProgress progress, boolean supportAutoIncrementKey, InsertOption insertOption, LookupResolutionType lookupResolutionType) - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to insert data into this table."); - - User updatedUser = getListUser(user, container); - DataIteratorContext context = new DataIteratorContext(errors); - context.setFailFast(false); - context.setInsertOption(insertOption); // this method is used by ListImporter and BackgroundListImporter - context.setSupportAutoIncrementKey(supportAutoIncrementKey); - context.setLookupResolutionType(lookupResolutionType); - setAttachmentDirectory(attachmentDir); - TableInfo ti = _list.getTable(updatedUser); - - if (null != ti) - { - try (DbScope.Transaction transaction = ti.getSchema().getScope().ensureTransaction()) - { - int imported = _importRowsUsingDIB(updatedUser, container, loader, null, context, new HashMap<>()); - - if (!errors.hasErrors()) - { - //Make entry to audit log if anything was inserted - if (imported > 0) - ListManager.get().addAuditEvent(_list, updatedUser, "Bulk " + (insertOption.updateOnly ? "updated " : (insertOption.mergeRows ? "imported " : "inserted ")) + imported + " rows to list."); - - transaction.addCommitTask(() -> ListManager.get().indexList(_list), DbScope.CommitTaskOption.POSTCOMMIT); - transaction.commit(); - - return imported; - } - - return 0; - } - } - - return 0; - } - - - @Override - public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, - @Nullable Map configParameters, Map extraScriptContext) - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to update data in this table."); - - return _importRowsUsingDIB(getListUser(user, container), container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); - } - - - @Override - public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, - Map configParameters, Map extraScriptContext) - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to insert data into this table."); - - DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); - int count = _importRowsUsingDIB(getListUser(user, container), container, rows, null, context, extraScriptContext); - if (count > 0 && !errors.hasErrors()) - ListManager.get().indexList(_list); - return count; - } - - @Override - public List> updateRows(User user, Container container, List> rows, List> oldKeys, - BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) - throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to update data into this table."); - - List> result = super.updateRows(getListUser(user, container), container, rows, oldKeys, errors, configParameters, extraScriptContext); - if (!result.isEmpty()) - ListManager.get().indexList(_list); - return result; - } - - @Override - protected Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, @Nullable Map configParameters) - throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException - { - // TODO: Check for equivalency so that attachments can be deleted etc. - - Map dps = new HashMap<>(); - for (DomainProperty dp : _list.getDomainOrThrow().getProperties()) - { - dps.put(dp.getPropertyURI(), dp); - } - - ValidatorContext validatorCache = new ValidatorContext(container, user); - - ListItm itm = new ListItm(); - itm.setEntityId((String) oldRow.get(ID)); - itm.setListId(_list.getListId()); - itm.setKey(oldRow.get(_list.getKeyName())); - - ListItem item = new ListItemImpl(_list, itm); - - if (item.getProperties() != null) - { - List errors = new ArrayList<>(); - for (Map.Entry entry : dps.entrySet()) - { - Object value = row.get(entry.getValue().getName()); - validateProperty(entry.getValue(), value, row, errors, validatorCache); - } - - if (!errors.isEmpty()) - throw new ValidationException(errors); - } - - // MVIndicators - Map rowCopy = new CaseInsensitiveHashMap<>(); - ArrayList modifiedAttachmentColumns = new ArrayList<>(); - ArrayList attachmentFiles = new ArrayList<>(); - - TableInfo qt = getQueryTable(); - for (Map.Entry r : row.entrySet()) - { - ColumnInfo column = qt.getColumn(FieldKey.fromParts(r.getKey())); - rowCopy.put(r.getKey(), r.getValue()); - - // 22747: Attachment columns - if (null != column) - { - DomainProperty dp = _list.getDomainOrThrow().getPropertyByURI(column.getPropertyURI()); - if (null != dp && isAttachmentProperty(dp)) - { - modifiedAttachmentColumns.add(column); - - // setup any new attachments - if (r.getValue() instanceof AttachmentFile file) - { - if (null != file.getFilename()) - attachmentFiles.add(file); - } - else if (r.getValue() != null && !StringUtils.isEmpty(String.valueOf(r.getValue()))) - { - throw new ValidationException("Can't upload '" + r.getValue() + "' to field " + r.getKey() + " with type Attachment."); - } - } - } - } - - // Attempt to include key from oldRow if not found in row (As stated in the QUS Interface) - Object newRowKey = getField(rowCopy, _list.getKeyName()); - Object oldRowKey = getField(oldRow, _list.getKeyName()); - - if (null == newRowKey && null != oldRowKey) - rowCopy.put(_list.getKeyName(), oldRowKey); - - Map result = super.updateRow(getListUser(user, container), container, rowCopy, oldRow, true, false); - - if (null != result) - { - result = getRow(user, container, result); - - if (null != result && null != result.get(ID)) - { - ListManager mgr = ListManager.get(); - String entityId = (String) result.get(ID); - - try - { - // Remove prior attachment -- only includes columns which are modified in this update - for (ColumnInfo col : modifiedAttachmentColumns) - { - Object value = oldRow.get(col.getName()); - if (null != value) - { - AttachmentService.get().deleteAttachment(new ListItemAttachmentParent(entityId, _list.getContainer()), value.toString(), user); - } - } - - // Update attachments - if (!attachmentFiles.isEmpty()) - AttachmentService.get().addAttachments(new ListItemAttachmentParent(entityId, _list.getContainer()), attachmentFiles, user); - } - catch (AttachmentService.DuplicateFilenameException | AttachmentService.FileTooLargeException e) - { - // issues 21503, 28633: turn these into a validation exception to get a nicer error - throw new ValidationException(e.getMessage()); - } - catch (IOException e) - { - throw UnexpectedException.wrap(e); - } - finally - { - for (AttachmentFile attachmentFile : attachmentFiles) - { - try { attachmentFile.closeInputStream(); } catch (IOException ignored) {} - } - } - - String oldRecord = mgr.formatAuditItem(_list, user, oldRow); - String newRecord = mgr.formatAuditItem(_list, user, result); - - // Audit - mgr.addAuditEvent(_list, user, container, "An existing list record was modified", entityId, oldRecord, newRecord); - } - } - - return result; - } - - // TODO: Consolidate with ColumnValidator and OntologyManager.validateProperty() - private boolean validateProperty(DomainProperty prop, Object value, Map newRow, List errors, ValidatorContext validatorCache) - { - //check for isRequired - if (prop.isRequired()) - { - // for mv indicator columns either an indicator or a field value is sufficient - boolean hasMvIndicator = prop.isMvEnabled() && (value instanceof ObjectProperty && ((ObjectProperty)value).getMvIndicator() != null); - if (!hasMvIndicator && (null == value || (value instanceof ObjectProperty && ((ObjectProperty)value).value() == null))) - { - if (newRow.containsKey(prop.getName()) && newRow.get(prop.getName()) == null) - { - errors.add(new PropertyValidationError("The field '" + prop.getName() + "' is required.", prop.getName())); - return false; - } - } - } - - if (null != value) - { - for (IPropertyValidator validator : prop.getValidators()) - { - if (!validator.validate(prop.getPropertyDescriptor(), value, errors, validatorCache)) - return false; - } - } - - return true; - } - - private record ListRecord(Object key, String entityId) { } - - @Override - public Map moveRows( - User _user, - Container container, - Container targetContainer, - List> rows, - BatchValidationException errors, - @Nullable Map configParameters, - @Nullable Map extraScriptContext - ) throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException - { - Map updateCounts = new HashMap<>(); - updateCounts.put("listAuditEventsCreated", 0); - updateCounts.put("listAuditEventsMoved", 0); - updateCounts.put("listRecords", 0); - updateCounts.put("queryAuditEventsMoved", 0); - - Map> containerRows = getListRowsForMoveRows(targetContainer, rows, errors); - if (errors.hasErrors() || containerRows.isEmpty()) - return updateCounts; - - AuditBehaviorType auditBehavior = configParameters != null ? (AuditBehaviorType) configParameters.get(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior) : null; - String auditUserComment = configParameters != null ? (String) configParameters.get(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment) : null; - User user = getListUser(_user, container); - String listSchemaName = ListSchema.getInstance().getSchemaName(); - boolean hasAttachmentProperties = _list.getDomainOrThrow() - .getProperties() - .stream() - .anyMatch(prop -> PropertyType.ATTACHMENT.equals(prop.getPropertyType())); - - int listAuditEventsCreatedCount = 0; - int listAuditEventsMovedCount = 0; - int listRecordsCount = 0; - int queryAuditEventsMovedCount = 0; - ListAuditProvider listAuditProvider = new ListAuditProvider(); - - try (DbScope.Transaction tx = getDbTable().getSchema().getScope().ensureTransaction()) - { - if (auditBehavior != null && AuditBehaviorType.NONE != auditBehavior && tx.getAuditEvent() == null) - { - TransactionAuditProvider.TransactionAuditEvent auditEvent = createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); - auditEvent.updateCommentRowCount(containerRows.values().stream().mapToInt(List::size).sum()); - AbstractQueryUpdateService.addTransactionAuditEvent(tx, user, auditEvent); - } - - List listAuditEvents = new ArrayList<>(); - - for (GUID containerId : containerRows.keySet()) - { - Container sourceContainer = ContainerManager.getForId(containerId); - if (sourceContainer == null) - throw new InvalidKeyException("Container '" + containerId + "' does not exist."); - - if (!sourceContainer.hasPermission(user, MoveEntitiesPermission.class)) - throw new UnauthorizedException("You do not have permission to move list records out of '" + sourceContainer.getName() + "'."); - - TableInfo listTable = _list.getTable(user, sourceContainer); - if (listTable == null) - throw new QueryUpdateServiceException(String.format("Failed to retrieve table for list '%s' in folder %s.", _list.getName(), sourceContainer.getPath())); - - List records = containerRows.get(containerId); - List rowPks = records.stream().map(ListRecord::key).toList(); - - Map extraContext = Map.of("targetContainer", targetContainer, "keys", rowPks); - listTable.fireBatchTrigger(sourceContainer, user, TableInfo.TriggerType.MOVE, true, errors, extraContext); - if (errors.hasErrors()) - return updateCounts; - - listRecordsCount += ContainerManager.updateContainer(getDbTable(), _list.getKeyName(), rowPks, targetContainer, user, true); - if (errors.hasErrors()) - return updateCounts; - - if (hasAttachmentProperties) - { - moveAttachments(user, sourceContainer, targetContainer, records, errors); - if (errors.hasErrors()) - return updateCounts; - } - - queryAuditEventsMovedCount += QueryService.get().moveAuditEvents(targetContainer, rowPks, listSchemaName, _list.getName()); - listAuditEventsMovedCount += listAuditProvider.moveEvents(targetContainer, records.stream().map(ListRecord::entityId).toList()); - - // Create a summary audit event for the source container - { - String comment = String.format("Moved %s to %s", StringUtilsLabKey.pluralize(records.size(), "row"), targetContainer.getPath()); - ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(sourceContainer, comment, _list); - event.setUserComment(auditUserComment); - listAuditEvents.add(event); - } - - // Create a summary audit event for the target container - { - String comment = String.format("Moved %s from %s", StringUtilsLabKey.pluralize(records.size(), "row"), sourceContainer.getPath()); - ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(targetContainer, comment, _list); - event.setUserComment(auditUserComment); - listAuditEvents.add(event); - } - - if (AuditBehaviorType.DETAILED == listTable.getEffectiveAuditBehavior(auditBehavior)) - listAuditEventsCreatedCount += addDetailedMoveAuditEvents(user, sourceContainer, targetContainer, records); - - // TODO: Picklist support? - - listTable.fireBatchTrigger(sourceContainer, user, TableInfo.TriggerType.MOVE, false, errors, extraContext); - if (errors.hasErrors()) - return updateCounts; - } - - if (!listAuditEvents.isEmpty()) - { - AuditLogService.get().addEvents(user, listAuditEvents, true); - listAuditEventsCreatedCount += listAuditEvents.size(); - } - - tx.addCommitTask(() -> ListManager.get().indexList(_list), DbScope.CommitTaskOption.POSTCOMMIT); - - tx.commit(); - - SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "moveEntities", "list"); - } - - updateCounts.put("listAuditEventsCreated", listAuditEventsCreatedCount); - updateCounts.put("listAuditEventsMoved", listAuditEventsMovedCount); - updateCounts.put("listRecords", listRecordsCount); - updateCounts.put("queryAuditEventsMoved", queryAuditEventsMovedCount); - - return updateCounts; - } - - private Map> getListRowsForMoveRows(Container targetContainer, List> rows, BatchValidationException errors) - { - if (rows.isEmpty()) - return Collections.emptyMap(); - - String keyName = _list.getKeyName(); - List keys = new ArrayList<>(); - for (var row : rows) - { - Object key = getField(row, keyName); - if (key == null) - { - errors.addRowError(new ValidationException("Key field value required for moving list rows.")); - return Collections.emptyMap(); - } - - keys.add(getKeyFilterValue(key)); - } - - SimpleFilter filter = new SimpleFilter(); - FieldKey fieldKey = FieldKey.fromParts(keyName); - filter.addInClause(fieldKey, keys); - filter.addCondition(FieldKey.fromParts("Container"), targetContainer.getId(), CompareType.NEQ); - - Map> containerRows = new HashMap<>(); - try (var result = new TableSelector(getQueryTable(), PageFlowUtil.set(keyName, "Container", "EntityId"), filter, null).getResults()) - { - while (result.next()) - { - GUID containerId = new GUID(result.getString("Container")); - containerRows.computeIfAbsent(containerId, k -> new ArrayList<>()); - containerRows.get(containerId).add(new ListRecord(result.getObject(fieldKey), result.getString("EntityId"))); - } - } - catch (SQLException e) - { - throw new RuntimeSQLException(e); - } - - return containerRows; - } - - private void moveAttachments(User user, Container sourceContainer, Container targetContainer, List records, BatchValidationException errors) - { - // TODO: Consider subsequent query to determine which rows need to be updated then only update those attachments - List parents = new ArrayList<>(); - for (ListRecord record : records) - parents.add(new ListItemAttachmentParent(record.entityId, sourceContainer)); - - try - { - AttachmentService.get().moveAttachments(targetContainer, parents, user); - } - catch (IOException e) - { - errors.addRowError(new ValidationException("Failed to move attachments when moving list rows. Error: " + e.getMessage())); - } - } - - private int addDetailedMoveAuditEvents(User user, Container sourceContainer, Container targetContainer, List records) - { - List auditEvents = new ArrayList<>(records.size()); - String keyName = _list.getKeyName(); - String sourcePath = sourceContainer.getPath(); - String targetPath = targetContainer.getPath(); - - for (ListRecord record : records) - { - ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(targetContainer, "An existing list record was moved", _list); - event.setListItemEntityId(record.entityId); - event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(CaseInsensitiveHashMap.of("Folder", sourcePath, keyName, record.key.toString()))); - event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(CaseInsensitiveHashMap.of("Folder", targetPath, keyName, record.key.toString()))); - auditEvents.add(event); - } - - AuditLogService.get().addEvents(user, auditEvents, true); - - return auditEvents.size(); - } - - @Override - protected Map deleteRow(User user, Container container, Map oldRowMap) throws InvalidKeyException, QueryUpdateServiceException, SQLException - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to delete data from this table."); - - Map result = super.deleteRow(getListUser(user, container), container, oldRowMap); - - if (null != result) - { - String entityId = (String) result.get(ID); - - if (null != entityId) - { - ListManager mgr = ListManager.get(); - String deletedRecord = mgr.formatAuditItem(_list, user, result); - - // Audit - mgr.addAuditEvent(_list, user, container, "An existing list record was deleted", entityId, deletedRecord, null); - - // Remove discussions - if (DiscussionService.get() != null) - DiscussionService.get().deleteDiscussions(container, user, entityId); - - // Remove attachments - if (hasAttachmentProperties()) - AttachmentService.get().deleteAttachments(new ListItemAttachmentParent(entityId, container)); - - // Clean up Search indexer - if (!result.isEmpty()) - mgr.deleteItemIndex(_list, entityId); - } - } - - return result; - } - - - // Deletes attachments & discussions, and removes list documents from full-text search index. - public void deleteRelatedListData(final User user, final Container container) - { - // Unindex all item docs and the entire list doc - ListManager.get().deleteIndexedList(_list); - - // Delete attachments and discussions associated with a list in batches of 1,000 - new TableSelector(getDbTable(), Collections.singleton("entityId")).forEachBatch(String.class, 1000, new ForEachBatchBlock<>() - { - @Override - public boolean accept(String entityId) - { - return null != entityId; - } - - @Override - public void exec(List entityIds) - { - // delete the related list data for this block - deleteRelatedListData(user, container, entityIds); - } - }); - } - - // delete the related list data for this block of entityIds - private void deleteRelatedListData(User user, Container container, List entityIds) - { - // Build up set of entityIds and AttachmentParents - List attachmentParents = new ArrayList<>(); - - // Delete Discussions - if (_list.getDiscussionSetting() != ListDefinition.DiscussionSetting.None && DiscussionService.get() != null) - DiscussionService.get().deleteDiscussions(container, user, entityIds); - - // Delete Attachments - if (hasAttachmentProperties()) - { - for (String entityId : entityIds) - { - attachmentParents.add(new ListItemAttachmentParent(entityId, container)); - } - AttachmentService.get().deleteAttachments(attachmentParents); - } - } - - @Override - protected int truncateRows(User user, Container container) throws QueryUpdateServiceException, SQLException - { - int result; - try (DbScope.Transaction transaction = getDbTable().getSchema().getScope().ensureTransaction()) - { - deleteRelatedListData(user, container); - result = super.truncateRows(getListUser(user, container), container); - transaction.addCommitTask(() -> ListManager.get().addAuditEvent(_list, user, "Deleted " + result + " rows from list."), DbScope.CommitTaskOption.POSTCOMMIT); - transaction.commit(); - } - - return result; - } - - @Nullable - public SimpleFilter getKeyFilter(Map map) throws InvalidKeyException - { - String keyName = _list.getKeyName(); - Object key = getField(map, keyName); - - if (null == key) - { - // Auto-increment lists might not provide a key so allow them to pass through - if (ListDefinition.KeyType.AutoIncrementInteger.equals(_list.getKeyType())) - return null; - throw new InvalidKeyException("No " + keyName + " provided for list \"" + _list.getName() + "\""); - } - - return new SimpleFilter(FieldKey.fromParts(keyName), getKeyFilterValue(key)); - } - - @NotNull - private Object getKeyFilterValue(@NotNull Object key) - { - ListDefinition.KeyType type = _list.getKeyType(); - - // Check the type of the list to ensure proper casting of the key type - if (ListDefinition.KeyType.Integer.equals(type) || ListDefinition.KeyType.AutoIncrementInteger.equals(type)) - return isIntegral(key) ? key : Integer.valueOf(key.toString()); - - return key.toString(); - } - - @Nullable - private Object getField(Map map, String key) - { - /* TODO: this is very strange, we have a TableInfo we should be using its ColumnInfo objects to figure out aliases, we don't need to guess */ - Object value = map.get(key); - - if (null == value) - value = map.get(key + "_"); - - if (null == value) - value = map.get(getDbTable().getSqlDialect().legalNameFromName(key)); - - return value; - } - - /** - * Delegate class to generate an AttachmentParent - */ - public static class ListItemAttachmentParentFactory implements AttachmentParentFactory - { - @Override - public AttachmentParent generateAttachmentParent(String entityId, Container c) - { - return new ListItemAttachmentParent(entityId, c); - } - } - - /** - * Get Domain from list definition, unless null then get from super - */ - @Override - protected Domain getDomain() - { - return _list != null? - _list.getDomain() : - super.getDomain(); - } -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.list.model; + +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.announcements.DiscussionService; +import org.labkey.api.attachments.AttachmentFile; +import org.labkey.api.attachments.AttachmentParent; +import org.labkey.api.attachments.AttachmentParentFactory; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.LookupResolutionType; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.Selector.ForEachBatchBlock; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; +import org.labkey.api.exp.ObjectProperty; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.list.ListDefinition; +import org.labkey.api.exp.list.ListImportProgress; +import org.labkey.api.exp.list.ListItem; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.IPropertyValidator; +import org.labkey.api.exp.property.ValidatorContext; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.lists.permissions.ManagePicklistsPermission; +import org.labkey.api.query.AbstractQueryUpdateService; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.DefaultQueryUpdateService; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.InvalidKeyException; +import org.labkey.api.query.PropertyValidationError; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateServiceException; +import org.labkey.api.query.ValidationError; +import org.labkey.api.query.ValidationException; +import org.labkey.api.reader.DataLoader; +import org.labkey.api.security.ElevatedUser; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.MoveEntitiesPermission; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.security.roles.EditorRole; +import org.labkey.api.usageMetrics.SimpleMetricsService; +import org.labkey.api.util.GUID; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.UnexpectedException; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.writer.VirtualFile; +import org.labkey.list.view.ListItemAttachmentParent; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.labkey.api.util.IntegerUtils.isIntegral; + +/** + * Implementation of QueryUpdateService for Lists + */ +public class ListQueryUpdateService extends DefaultQueryUpdateService +{ + private final ListDefinitionImpl _list; + private static final String ID = "entityId"; + + public ListQueryUpdateService(ListTable queryTable, TableInfo dbTable, @NotNull ListDefinition list) + { + super(queryTable, dbTable, createMVMapping(queryTable.getList().getDomain())); + _list = (ListDefinitionImpl) list; + } + + @Override + public void configureDataIteratorContext(DataIteratorContext context) + { + if (context.getInsertOption().batch) + { + context.setMaxRowErrors(100); + context.setFailFast(false); + } + + context.putConfigParameter(ConfigParameters.TrimStringRight, Boolean.TRUE); + } + + @Override + protected @Nullable AttachmentParentFactory getAttachmentParentFactory() + { + return new ListItemAttachmentParentFactory(); + } + + @Override + protected Map getRow(User user, Container container, Map listRow) throws InvalidKeyException + { + Map ret = null; + + if (null != listRow) + { + SimpleFilter keyFilter = getKeyFilter(listRow); + + if (null != keyFilter) + { + TableInfo queryTable = getQueryTable(); + Map raw = new TableSelector(queryTable, keyFilter, null).getMap(); + + if (null != raw && !raw.isEmpty()) + { + ret = new CaseInsensitiveHashMap<>(); + + // EntityId + ret.put("EntityId", raw.get("entityid")); + + for (DomainProperty prop : _list.getDomainOrThrow().getProperties()) + { + String propName = prop.getName(); + ColumnInfo column = queryTable.getColumn(propName); + Object value = column.getValue(raw); + if (value != null) + ret.put(propName, value); + } + } + } + } + + return ret; + } + + + @Override + public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, + @Nullable Map configParameters, Map extraScriptContext) + { + for (Map row : rows) + { + aliasColumns(getColumnMapping(), row); + } + + DataIteratorContext context = getDataIteratorContext(errors, InsertOption.INSERT, configParameters); + List> result = this._insertRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); + + if (null != result) + { + ListManager mgr = ListManager.get(); + + for (Map row : result) + { + if (null != row.get(ID)) + { + // Audit each row + String entityId = (String) row.get(ID); + String newRecord = mgr.formatAuditItem(_list, user, row); + + mgr.addAuditEvent(_list, user, container, "A new list record was inserted", entityId, null, newRecord); + } + } + + if (!result.isEmpty() && !errors.hasErrors()) + mgr.indexList(_list); + } + + return result; + } + + private User getListUser(User user, Container container) + { + if (_list.isPicklist() && container.hasPermission(user, ManagePicklistsPermission.class)) + { + // if the list is a picklist and you have permission to manage picklists, that equates + // to having editor permission. + return ElevatedUser.ensureContextualRoles(container, user, Pair.of(DeletePermission.class, EditorRole.class)); + } + return user; + } + + @Override + protected @Nullable List> _insertRowsUsingDIB(User user, Container container, List> rows, DataIteratorContext context, @Nullable Map extraScriptContext) + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to insert data into this table."); + + return super._insertRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); + } + + @Override + protected @Nullable List> _updateRowsUsingDIB(User user, Container container, List> rows, + DataIteratorContext context, @Nullable Map extraScriptContext) + { + if (!hasPermission(user, UpdatePermission.class)) + throw new UnauthorizedException("You do not have permission to update data in this table."); + + return super._updateRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); + } + + public int insertUsingDataIterator(DataLoader loader, User user, Container container, BatchValidationException errors, @Nullable VirtualFile attachmentDir, + @Nullable ListImportProgress progress, boolean supportAutoIncrementKey, InsertOption insertOption, LookupResolutionType lookupResolutionType) + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to insert data into this table."); + + User updatedUser = getListUser(user, container); + DataIteratorContext context = new DataIteratorContext(errors); + context.setFailFast(false); + context.setInsertOption(insertOption); // this method is used by ListImporter and BackgroundListImporter + context.setSupportAutoIncrementKey(supportAutoIncrementKey); + context.setLookupResolutionType(lookupResolutionType); + setAttachmentDirectory(attachmentDir); + TableInfo ti = _list.getTable(updatedUser); + + if (null != ti) + { + try (DbScope.Transaction transaction = ti.getSchema().getScope().ensureTransaction()) + { + int imported = _importRowsUsingDIB(updatedUser, container, loader, null, context, new HashMap<>()); + + if (!errors.hasErrors()) + { + //Make entry to audit log if anything was inserted + if (imported > 0) + ListManager.get().addAuditEvent(_list, updatedUser, "Bulk " + (insertOption.updateOnly ? "updated " : (insertOption.mergeRows ? "imported " : "inserted ")) + imported + " rows to list."); + + transaction.addCommitTask(() -> ListManager.get().indexList(_list), DbScope.CommitTaskOption.POSTCOMMIT); + transaction.commit(); + + return imported; + } + + return 0; + } + } + + return 0; + } + + + @Override + public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, + @Nullable Map configParameters, Map extraScriptContext) + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to update data in this table."); + + return _importRowsUsingDIB(getListUser(user, container), container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); + } + + + @Override + public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, + Map configParameters, Map extraScriptContext) + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to insert data into this table."); + + DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); + int count = _importRowsUsingDIB(getListUser(user, container), container, rows, null, context, extraScriptContext); + if (count > 0 && !errors.hasErrors()) + ListManager.get().indexList(_list); + return count; + } + + @Override + public List> updateRows(User user, Container container, List> rows, List> oldKeys, + BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) + throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to update data into this table."); + + List> result = super.updateRows(getListUser(user, container), container, rows, oldKeys, errors, configParameters, extraScriptContext); + if (!result.isEmpty()) + ListManager.get().indexList(_list); + return result; + } + + @Override + protected Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, @Nullable Map configParameters) + throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException + { + // TODO: Check for equivalency so that attachments can be deleted etc. + + Map dps = new HashMap<>(); + for (DomainProperty dp : _list.getDomainOrThrow().getProperties()) + { + dps.put(dp.getPropertyURI(), dp); + } + + ValidatorContext validatorCache = new ValidatorContext(container, user); + + ListItm itm = new ListItm(); + itm.setEntityId((String) oldRow.get(ID)); + itm.setListId(_list.getListId()); + itm.setKey(oldRow.get(_list.getKeyName())); + + ListItem item = new ListItemImpl(_list, itm); + + if (item.getProperties() != null) + { + List errors = new ArrayList<>(); + for (Map.Entry entry : dps.entrySet()) + { + Object value = row.get(entry.getValue().getName()); + validateProperty(entry.getValue(), value, row, errors, validatorCache); + } + + if (!errors.isEmpty()) + throw new ValidationException(errors); + } + + // MVIndicators + Map rowCopy = new CaseInsensitiveHashMap<>(); + ArrayList modifiedAttachmentColumns = new ArrayList<>(); + ArrayList attachmentFiles = new ArrayList<>(); + + TableInfo qt = getQueryTable(); + for (Map.Entry r : row.entrySet()) + { + ColumnInfo column = qt.getColumn(FieldKey.fromParts(r.getKey())); + rowCopy.put(r.getKey(), r.getValue()); + + // 22747: Attachment columns + if (null != column) + { + DomainProperty dp = _list.getDomainOrThrow().getPropertyByURI(column.getPropertyURI()); + if (null != dp && isAttachmentProperty(dp)) + { + modifiedAttachmentColumns.add(column); + + // setup any new attachments + if (r.getValue() instanceof AttachmentFile file) + { + if (null != file.getFilename()) + attachmentFiles.add(file); + } + else if (r.getValue() != null && !StringUtils.isEmpty(String.valueOf(r.getValue()))) + { + throw new ValidationException("Can't upload '" + r.getValue() + "' to field " + r.getKey() + " with type Attachment."); + } + } + } + } + + // Attempt to include key from oldRow if not found in row (As stated in the QUS Interface) + Object newRowKey = getField(rowCopy, _list.getKeyName()); + Object oldRowKey = getField(oldRow, _list.getKeyName()); + + if (null == newRowKey && null != oldRowKey) + rowCopy.put(_list.getKeyName(), oldRowKey); + + Map result = super.updateRow(getListUser(user, container), container, rowCopy, oldRow, true, false); + + if (null != result) + { + result = getRow(user, container, result); + + if (null != result && null != result.get(ID)) + { + ListManager mgr = ListManager.get(); + String entityId = (String) result.get(ID); + + try + { + // Remove prior attachment -- only includes columns which are modified in this update + for (ColumnInfo col : modifiedAttachmentColumns) + { + Object value = oldRow.get(col.getName()); + if (null != value) + { + AttachmentService.get().deleteAttachment(new ListItemAttachmentParent(entityId, _list.getContainer()), value.toString(), user); + } + } + + // Update attachments + if (!attachmentFiles.isEmpty()) + AttachmentService.get().addAttachments(new ListItemAttachmentParent(entityId, _list.getContainer()), attachmentFiles, user); + } + catch (AttachmentService.DuplicateFilenameException | AttachmentService.FileTooLargeException e) + { + // issues 21503, 28633: turn these into a validation exception to get a nicer error + throw new ValidationException(e.getMessage()); + } + catch (IOException e) + { + throw UnexpectedException.wrap(e); + } + finally + { + for (AttachmentFile attachmentFile : attachmentFiles) + { + try { attachmentFile.closeInputStream(); } catch (IOException ignored) {} + } + } + + String oldRecord = mgr.formatAuditItem(_list, user, oldRow); + String newRecord = mgr.formatAuditItem(_list, user, result); + + // Audit + mgr.addAuditEvent(_list, user, container, "An existing list record was modified", entityId, oldRecord, newRecord); + } + } + + return result; + } + + // TODO: Consolidate with ColumnValidator and OntologyManager.validateProperty() + private boolean validateProperty(DomainProperty prop, Object value, Map newRow, List errors, ValidatorContext validatorCache) + { + //check for isRequired + if (prop.isRequired()) + { + // for mv indicator columns either an indicator or a field value is sufficient + boolean hasMvIndicator = prop.isMvEnabled() && (value instanceof ObjectProperty && ((ObjectProperty)value).getMvIndicator() != null); + if (!hasMvIndicator && (null == value || (value instanceof ObjectProperty && ((ObjectProperty)value).value() == null))) + { + if (newRow.containsKey(prop.getName()) && newRow.get(prop.getName()) == null) + { + errors.add(new PropertyValidationError("The field '" + prop.getName() + "' is required.", prop.getName())); + return false; + } + } + } + + if (null != value) + { + for (IPropertyValidator validator : prop.getValidators()) + { + if (!validator.validate(prop.getPropertyDescriptor(), value, errors, validatorCache)) + return false; + } + } + + return true; + } + + private record ListRecord(Object key, String entityId) { } + + @Override + public Map moveRows( + User _user, + Container container, + Container targetContainer, + List> rows, + BatchValidationException errors, + @Nullable Map configParameters, + @Nullable Map extraScriptContext + ) throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException + { + Map> containerRows = getListRowsForMoveRows(targetContainer, rows, errors); + if (errors.hasErrors()) + throw errors; + + int listAuditEventsCreatedCount = 0; + int listAuditEventsMovedCount = 0; + int listRecordsCount = 0; + int queryAuditEventsMovedCount = 0; + + if (containerRows.isEmpty()) + return moveRowsCounts(listAuditEventsCreatedCount, listAuditEventsMovedCount, listRecordsCount, queryAuditEventsMovedCount); + + AuditBehaviorType auditBehavior = configParameters != null ? (AuditBehaviorType) configParameters.get(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior) : null; + String auditUserComment = configParameters != null ? (String) configParameters.get(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment) : null; + User user = getListUser(_user, container); + String listSchemaName = ListSchema.getInstance().getSchemaName(); + boolean hasAttachmentProperties = _list.getDomainOrThrow() + .getProperties() + .stream() + .anyMatch(prop -> PropertyType.ATTACHMENT.equals(prop.getPropertyType())); + + ListAuditProvider listAuditProvider = new ListAuditProvider(); + + try (DbScope.Transaction tx = getDbTable().getSchema().getScope().ensureTransaction()) + { + if (auditBehavior != null && AuditBehaviorType.NONE != auditBehavior && tx.getAuditEvent() == null) + { + TransactionAuditProvider.TransactionAuditEvent auditEvent = createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); + auditEvent.updateCommentRowCount(containerRows.values().stream().mapToInt(List::size).sum()); + AbstractQueryUpdateService.addTransactionAuditEvent(tx, user, auditEvent); + } + + List listAuditEvents = new ArrayList<>(); + + for (GUID containerId : containerRows.keySet()) + { + Container sourceContainer = ContainerManager.getForId(containerId); + if (sourceContainer == null) + throw new InvalidKeyException("Container '" + containerId + "' does not exist."); + + if (!sourceContainer.hasPermission(user, MoveEntitiesPermission.class)) + throw new UnauthorizedException("You do not have permission to move list records out of '" + sourceContainer.getName() + "'."); + + TableInfo listTable = _list.getTable(user, sourceContainer); + if (listTable == null) + throw new QueryUpdateServiceException(String.format("Failed to retrieve table for list '%s' in folder %s.", _list.getName(), sourceContainer.getPath())); + + List records = containerRows.get(containerId); + List rowPks = records.stream().map(ListRecord::key).toList(); + + Map extraContext = Map.of("targetContainer", targetContainer, "keys", rowPks); + listTable.fireBatchTrigger(sourceContainer, user, TableInfo.TriggerType.MOVE, true, errors, extraContext); + if (errors.hasErrors()) + throw errors; + + listRecordsCount += ContainerManager.updateContainer(getDbTable(), _list.getKeyName(), rowPks, targetContainer, user, true); + if (errors.hasErrors()) + throw errors; + + if (hasAttachmentProperties) + { + moveAttachments(user, sourceContainer, targetContainer, records, errors); + if (errors.hasErrors()) + throw errors; + } + + queryAuditEventsMovedCount += QueryService.get().moveAuditEvents(targetContainer, rowPks, listSchemaName, _list.getName()); + listAuditEventsMovedCount += listAuditProvider.moveEvents(targetContainer, records.stream().map(ListRecord::entityId).toList()); + + // Create a summary audit event for the source container + { + String comment = String.format("Moved %s to %s", StringUtilsLabKey.pluralize(records.size(), "row"), targetContainer.getPath()); + ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(sourceContainer, comment, _list); + event.setUserComment(auditUserComment); + listAuditEvents.add(event); + } + + // Create a summary audit event for the target container + { + String comment = String.format("Moved %s from %s", StringUtilsLabKey.pluralize(records.size(), "row"), sourceContainer.getPath()); + ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(targetContainer, comment, _list); + event.setUserComment(auditUserComment); + listAuditEvents.add(event); + } + + if (AuditBehaviorType.DETAILED == listTable.getEffectiveAuditBehavior(auditBehavior)) + listAuditEventsCreatedCount += addDetailedMoveAuditEvents(user, sourceContainer, targetContainer, records); + + // TODO: Picklist support? + + listTable.fireBatchTrigger(sourceContainer, user, TableInfo.TriggerType.MOVE, false, errors, extraContext); + if (errors.hasErrors()) + throw errors; + } + + if (!listAuditEvents.isEmpty()) + { + AuditLogService.get().addEvents(user, listAuditEvents, true); + listAuditEventsCreatedCount += listAuditEvents.size(); + } + + tx.addCommitTask(() -> ListManager.get().indexList(_list), DbScope.CommitTaskOption.POSTCOMMIT); + + tx.commit(); + + SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "moveEntities", "list"); + } + + return moveRowsCounts(listAuditEventsCreatedCount, listAuditEventsMovedCount, listRecordsCount, queryAuditEventsMovedCount); + } + + private Map moveRowsCounts(int listAuditEventsCreated, int listAuditEventsMoved, int listRecords, int queryAuditEventsMoved) + { + return Map.of( + "listAuditEventsCreated", listAuditEventsCreated, + "listAuditEventsMoved", listAuditEventsMoved, + "listRecords", listRecords, + "queryAuditEventsMoved", queryAuditEventsMoved + ); + } + + private Map> getListRowsForMoveRows(Container targetContainer, List> rows, BatchValidationException errors) + { + if (rows.isEmpty()) + return Collections.emptyMap(); + + String keyName = _list.getKeyName(); + List keys = new ArrayList<>(); + for (var row : rows) + { + Object key = getField(row, keyName); + if (key == null) + { + errors.addRowError(new ValidationException("Key field value required for moving list rows.")); + return Collections.emptyMap(); + } + + keys.add(getKeyFilterValue(key)); + } + + SimpleFilter filter = new SimpleFilter(); + FieldKey fieldKey = FieldKey.fromParts(keyName); + filter.addInClause(fieldKey, keys); + filter.addCondition(FieldKey.fromParts("Container"), targetContainer.getId(), CompareType.NEQ); + + Map> containerRows = new HashMap<>(); + try (var result = new TableSelector(getQueryTable(), PageFlowUtil.set(keyName, "Container", "EntityId"), filter, null).getResults()) + { + while (result.next()) + { + GUID containerId = new GUID(result.getString("Container")); + containerRows.computeIfAbsent(containerId, k -> new ArrayList<>()); + containerRows.get(containerId).add(new ListRecord(result.getObject(fieldKey), result.getString("EntityId"))); + } + } + catch (SQLException e) + { + throw new RuntimeSQLException(e); + } + + return containerRows; + } + + private void moveAttachments(User user, Container sourceContainer, Container targetContainer, List records, BatchValidationException errors) + { + // TODO: Consider subsequent query to determine which rows need to be updated then only update those attachments + List parents = new ArrayList<>(); + for (ListRecord record : records) + parents.add(new ListItemAttachmentParent(record.entityId, sourceContainer)); + + try + { + AttachmentService.get().moveAttachments(targetContainer, parents, user); + } + catch (IOException e) + { + errors.addRowError(new ValidationException("Failed to move attachments when moving list rows. Error: " + e.getMessage())); + } + } + + private int addDetailedMoveAuditEvents(User user, Container sourceContainer, Container targetContainer, List records) + { + List auditEvents = new ArrayList<>(records.size()); + String keyName = _list.getKeyName(); + String sourcePath = sourceContainer.getPath(); + String targetPath = targetContainer.getPath(); + + for (ListRecord record : records) + { + ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(targetContainer, "An existing list record was moved", _list); + event.setListItemEntityId(record.entityId); + event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(CaseInsensitiveHashMap.of("Folder", sourcePath, keyName, record.key.toString()))); + event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(CaseInsensitiveHashMap.of("Folder", targetPath, keyName, record.key.toString()))); + auditEvents.add(event); + } + + AuditLogService.get().addEvents(user, auditEvents, true); + + return auditEvents.size(); + } + + @Override + protected Map deleteRow(User user, Container container, Map oldRowMap) throws InvalidKeyException, QueryUpdateServiceException, SQLException + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to delete data from this table."); + + Map result = super.deleteRow(getListUser(user, container), container, oldRowMap); + + if (null != result) + { + String entityId = (String) result.get(ID); + + if (null != entityId) + { + ListManager mgr = ListManager.get(); + String deletedRecord = mgr.formatAuditItem(_list, user, result); + + // Audit + mgr.addAuditEvent(_list, user, container, "An existing list record was deleted", entityId, deletedRecord, null); + + // Remove discussions + if (DiscussionService.get() != null) + DiscussionService.get().deleteDiscussions(container, user, entityId); + + // Remove attachments + if (hasAttachmentProperties()) + AttachmentService.get().deleteAttachments(new ListItemAttachmentParent(entityId, container)); + + // Clean up Search indexer + if (!result.isEmpty()) + mgr.deleteItemIndex(_list, entityId); + } + } + + return result; + } + + + // Deletes attachments & discussions, and removes list documents from full-text search index. + public void deleteRelatedListData(final User user, final Container container) + { + // Unindex all item docs and the entire list doc + ListManager.get().deleteIndexedList(_list); + + // Delete attachments and discussions associated with a list in batches of 1,000 + new TableSelector(getDbTable(), Collections.singleton("entityId")).forEachBatch(String.class, 1000, new ForEachBatchBlock<>() + { + @Override + public boolean accept(String entityId) + { + return null != entityId; + } + + @Override + public void exec(List entityIds) + { + // delete the related list data for this block + deleteRelatedListData(user, container, entityIds); + } + }); + } + + // delete the related list data for this block of entityIds + private void deleteRelatedListData(User user, Container container, List entityIds) + { + // Build up set of entityIds and AttachmentParents + List attachmentParents = new ArrayList<>(); + + // Delete Discussions + if (_list.getDiscussionSetting() != ListDefinition.DiscussionSetting.None && DiscussionService.get() != null) + DiscussionService.get().deleteDiscussions(container, user, entityIds); + + // Delete Attachments + if (hasAttachmentProperties()) + { + for (String entityId : entityIds) + { + attachmentParents.add(new ListItemAttachmentParent(entityId, container)); + } + AttachmentService.get().deleteAttachments(attachmentParents); + } + } + + @Override + protected int truncateRows(User user, Container container) throws QueryUpdateServiceException, SQLException + { + int result; + try (DbScope.Transaction transaction = getDbTable().getSchema().getScope().ensureTransaction()) + { + deleteRelatedListData(user, container); + result = super.truncateRows(getListUser(user, container), container); + transaction.addCommitTask(() -> ListManager.get().addAuditEvent(_list, user, "Deleted " + result + " rows from list."), DbScope.CommitTaskOption.POSTCOMMIT); + transaction.commit(); + } + + return result; + } + + @Nullable + public SimpleFilter getKeyFilter(Map map) throws InvalidKeyException + { + String keyName = _list.getKeyName(); + Object key = getField(map, keyName); + + if (null == key) + { + // Auto-increment lists might not provide a key so allow them to pass through + if (ListDefinition.KeyType.AutoIncrementInteger.equals(_list.getKeyType())) + return null; + throw new InvalidKeyException("No " + keyName + " provided for list \"" + _list.getName() + "\""); + } + + return new SimpleFilter(FieldKey.fromParts(keyName), getKeyFilterValue(key)); + } + + @NotNull + private Object getKeyFilterValue(@NotNull Object key) + { + ListDefinition.KeyType type = _list.getKeyType(); + + // Check the type of the list to ensure proper casting of the key type + if (ListDefinition.KeyType.Integer.equals(type) || ListDefinition.KeyType.AutoIncrementInteger.equals(type)) + return isIntegral(key) ? key : Integer.valueOf(key.toString()); + + return key.toString(); + } + + @Nullable + private Object getField(Map map, String key) + { + /* TODO: this is very strange, we have a TableInfo we should be using its ColumnInfo objects to figure out aliases, we don't need to guess */ + Object value = map.get(key); + + if (null == value) + value = map.get(key + "_"); + + if (null == value) + value = map.get(getDbTable().getSqlDialect().legalNameFromName(key)); + + return value; + } + + /** + * Delegate class to generate an AttachmentParent + */ + public static class ListItemAttachmentParentFactory implements AttachmentParentFactory + { + @Override + public AttachmentParent generateAttachmentParent(String entityId, Container c) + { + return new ListItemAttachmentParent(entityId, c); + } + } + + /** + * Get Domain from list definition, unless null then get from super + */ + @Override + protected Domain getDomain() + { + return _list != null? + _list.getDomain() : + super.getDomain(); + } +} diff --git a/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java b/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java index 2de865a1aca..3e2efec92bd 100644 --- a/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java +++ b/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java @@ -1,311 +1,307 @@ -/* - * Copyright (c) 2013-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.query.audit; - -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.AuditTypeProvider; -import org.labkey.api.audit.DetailedAuditTypeEvent; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.query.AbstractAuditDomainKind; -import org.labkey.api.audit.query.DefaultAuditTypeTable; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.MutableColumnInfo; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.TableInfo; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryForm; -import org.labkey.api.query.QuerySettings; -import org.labkey.api.query.QueryView; -import org.labkey.api.query.UserSchema; -import org.labkey.api.view.ViewContext; -import org.labkey.query.controllers.QueryController; -import org.springframework.validation.BindException; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * User: klum - * Date: 7/21/13 - */ -public class QueryUpdateAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider -{ - public static final String QUERY_UPDATE_AUDIT_EVENT = "QueryUpdateAuditEvent"; - - public static final String COLUMN_NAME_ROW_PK = "RowPk"; - public static final String COLUMN_NAME_SCHEMA_NAME = "SchemaName"; - public static final String COLUMN_NAME_QUERY_NAME = "QueryName"; - - static final List defaultVisibleColumns = new ArrayList<>(); - - static { - - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED_BY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_IMPERSONATED_BY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_PROJECT_ID)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CONTAINER)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_SCHEMA_NAME)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_QUERY_NAME)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_COMMENT)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_USER_COMMENT)); - } - - public QueryUpdateAuditProvider() - { - super(new QueryUpdateAuditDomainKind()); - } - - @Override - public String getEventName() - { - return QUERY_UPDATE_AUDIT_EVENT; - } - - @Override - public String getLabel() - { - return "Query update events"; - } - - @Override - public String getDescription() - { - return "Data about insert and update queries."; - } - - @Override - public Map legacyNameMap() - { - Map legacyMap = super.legacyNameMap(); - legacyMap.put(FieldKey.fromParts("key1"), COLUMN_NAME_ROW_PK); - legacyMap.put(FieldKey.fromParts("key2"), COLUMN_NAME_SCHEMA_NAME); - legacyMap.put(FieldKey.fromParts("key3"), COLUMN_NAME_QUERY_NAME); - legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.OLD_RECORD_PROP_NAME), AbstractAuditDomainKind.OLD_RECORD_PROP_NAME); - legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.NEW_RECORD_PROP_NAME), AbstractAuditDomainKind.NEW_RECORD_PROP_NAME); - return legacyMap; - } - - @Override - public Class getEventClass() - { - return (Class)QueryUpdateAuditEvent.class; - } - - @Override - public TableInfo createTableInfo(UserSchema userSchema, ContainerFilter cf) - { - DefaultAuditTypeTable table = new DefaultAuditTypeTable(this, createStorageTableInfo(), userSchema, cf, defaultVisibleColumns) - { - @Override - protected void initColumn(MutableColumnInfo col) - { - if (COLUMN_NAME_SCHEMA_NAME.equalsIgnoreCase(col.getName())) - { - col.setLabel("Schema Name"); - } - else if (COLUMN_NAME_QUERY_NAME.equalsIgnoreCase(col.getName())) - { - col.setLabel("Query Name"); - } - else if (COLUMN_NAME_USER_COMMENT.equalsIgnoreCase(col.getName())) - { - col.setLabel("User Comment"); - } - } - }; - appendValueMapColumns(table, QUERY_UPDATE_AUDIT_EVENT); - - return table; - } - - @Override - public List getDefaultVisibleColumns() - { - return defaultVisibleColumns; - } - - - public static QueryView createHistoryQueryView(ViewContext context, QueryForm form, BindException errors) - { - UserSchema schema = AuditLogService.getAuditLogSchema(context.getUser(), context.getContainer()); - if (schema != null) - { - QuerySettings settings = new QuerySettings(context, QueryView.DATAREGIONNAME_DEFAULT); - - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_SCHEMA_NAME), form.getSchemaName()); - filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_QUERY_NAME), form.getQueryName()); - - settings.setBaseFilter(filter); - settings.setQueryName(QUERY_UPDATE_AUDIT_EVENT); - return schema.createView(context, settings, errors); - } - return null; - } - - public static QueryView createDetailsQueryView(ViewContext context, QueryController.QueryDetailsForm form, BindException errors) - { - return createDetailsQueryView(context, form.getSchemaName(), form.getQueryName(), form.getKeyValue(), errors); - } - - public static QueryView createDetailsQueryView(ViewContext context, String schemaName, String queryName, String keyValue, BindException errors) - { - UserSchema schema = AuditLogService.getAuditLogSchema(context.getUser(), context.getContainer()); - if (schema != null) - { - QuerySettings settings = new QuerySettings(context, QueryView.DATAREGIONNAME_DEFAULT); - - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_SCHEMA_NAME), schemaName); - filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_QUERY_NAME), queryName); - filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_ROW_PK), keyValue); - - settings.setBaseFilter(filter); - settings.setQueryName(QUERY_UPDATE_AUDIT_EVENT); - return schema.createView(context, settings, errors); - } - return null; - } - - public int moveEvents(Container targetContainer, Collection rowPks, String schemaName, String queryName) - { - TableInfo auditTable = createStorageTableInfo(); - SQLFragment sql = new SQLFragment("UPDATE ").append(auditTable) - .append(" SET container = ").appendValue(targetContainer) - .append(" WHERE RowPk "); - auditTable.getSchema().getSqlDialect().appendInClauseSql(sql, rowPks.stream().map(Object::toString).toList()); - sql.append(" AND SchemaName = ").appendValue(schemaName).append(" AND QueryName = ").appendValue(queryName); - return new SqlExecutor(auditTable.getSchema()).execute(sql); - } - - public static class QueryUpdateAuditEvent extends DetailedAuditTypeEvent - { - private String _rowPk; - private String _schemaName; - private String _queryName; - - /** Important for reflection-based instantiation */ - @SuppressWarnings("unused") - public QueryUpdateAuditEvent() - { - super(); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); - } - - public QueryUpdateAuditEvent(Container container, String comment) - { - super(QUERY_UPDATE_AUDIT_EVENT, container, comment); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); - } - - public String getRowPk() - { - return _rowPk; - } - - public void setRowPk(String rowPk) - { - _rowPk = rowPk; - } - - public String getSchemaName() - { - return _schemaName; - } - - public void setSchemaName(String schemaName) - { - _schemaName = schemaName; - } - - public String getQueryName() - { - return _queryName; - } - - public void setQueryName(String queryName) - { - _queryName = queryName; - } - - @Override - public Map getAuditLogMessageElements() - { - Map elements = new LinkedHashMap<>(); - elements.put("rowPk", getRowPk()); - elements.put("schemaName", getSchemaName()); - elements.put("queryName", getQueryName()); - elements.put("transactionId", getTransactionId()); - elements.put("userComment", getUserComment()); - // N.B. oldRecordMap and newRecordMap are potentially very large (and are not displayed in the default grid view) - elements.putAll(super.getAuditLogMessageElements()); - return elements; - } - } - - public static class QueryUpdateAuditDomainKind extends AbstractAuditDomainKind - { - public static final String NAME = "QueryUpdateAuditDomain"; - public static String NAMESPACE_PREFIX = "Audit-" + NAME; - - private final Set _fields; - - public QueryUpdateAuditDomainKind() - { - super(QUERY_UPDATE_AUDIT_EVENT); - - Set fields = new LinkedHashSet<>(); - fields.add(createPropertyDescriptor(COLUMN_NAME_ROW_PK, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_SCHEMA_NAME, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_QUERY_NAME, PropertyType.STRING)); - fields.add(createOldDataMapPropertyDescriptor()); - fields.add(createNewDataMapPropertyDescriptor()); - fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_ID, PropertyType.BIGINT)); - fields.add(createPropertyDescriptor(COLUMN_NAME_USER_COMMENT, PropertyType.STRING)); - _fields = Collections.unmodifiableSet(fields); - } - - @Override - public Set getProperties() - { - return _fields; - } - - @Override - protected String getNamespacePrefix() - { - return NAMESPACE_PREFIX; - } - - @Override - public String getKindName() - { - return NAME; - } - } -} +/* + * Copyright (c) 2013-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.query.audit; + +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.AuditTypeProvider; +import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.query.AbstractAuditDomainKind; +import org.labkey.api.audit.query.DefaultAuditTypeTable; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.MutableColumnInfo; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.TableInfo; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryForm; +import org.labkey.api.query.QuerySettings; +import org.labkey.api.query.QueryView; +import org.labkey.api.query.UserSchema; +import org.labkey.api.view.ViewContext; +import org.labkey.query.controllers.QueryController; +import org.springframework.validation.BindException; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class QueryUpdateAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider +{ + public static final String QUERY_UPDATE_AUDIT_EVENT = "QueryUpdateAuditEvent"; + + public static final String COLUMN_NAME_ROW_PK = "RowPk"; + public static final String COLUMN_NAME_SCHEMA_NAME = "SchemaName"; + public static final String COLUMN_NAME_QUERY_NAME = "QueryName"; + + static final List defaultVisibleColumns = new ArrayList<>(); + + static { + + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED_BY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_IMPERSONATED_BY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_PROJECT_ID)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CONTAINER)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_SCHEMA_NAME)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_QUERY_NAME)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_COMMENT)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_USER_COMMENT)); + } + + public QueryUpdateAuditProvider() + { + super(new QueryUpdateAuditDomainKind()); + } + + @Override + public String getEventName() + { + return QUERY_UPDATE_AUDIT_EVENT; + } + + @Override + public String getLabel() + { + return "Query update events"; + } + + @Override + public String getDescription() + { + return "Data about insert and update queries."; + } + + @Override + public Map legacyNameMap() + { + Map legacyMap = super.legacyNameMap(); + legacyMap.put(FieldKey.fromParts("key1"), COLUMN_NAME_ROW_PK); + legacyMap.put(FieldKey.fromParts("key2"), COLUMN_NAME_SCHEMA_NAME); + legacyMap.put(FieldKey.fromParts("key3"), COLUMN_NAME_QUERY_NAME); + legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.OLD_RECORD_PROP_NAME), AbstractAuditDomainKind.OLD_RECORD_PROP_NAME); + legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.NEW_RECORD_PROP_NAME), AbstractAuditDomainKind.NEW_RECORD_PROP_NAME); + return legacyMap; + } + + @Override + public Class getEventClass() + { + return (Class)QueryUpdateAuditEvent.class; + } + + @Override + public TableInfo createTableInfo(UserSchema userSchema, ContainerFilter cf) + { + DefaultAuditTypeTable table = new DefaultAuditTypeTable(this, createStorageTableInfo(), userSchema, cf, defaultVisibleColumns) + { + @Override + protected void initColumn(MutableColumnInfo col) + { + if (COLUMN_NAME_SCHEMA_NAME.equalsIgnoreCase(col.getName())) + { + col.setLabel("Schema Name"); + } + else if (COLUMN_NAME_QUERY_NAME.equalsIgnoreCase(col.getName())) + { + col.setLabel("Query Name"); + } + else if (COLUMN_NAME_USER_COMMENT.equalsIgnoreCase(col.getName())) + { + col.setLabel("User Comment"); + } + } + }; + appendValueMapColumns(table, QUERY_UPDATE_AUDIT_EVENT); + + return table; + } + + @Override + public List getDefaultVisibleColumns() + { + return defaultVisibleColumns; + } + + + public static QueryView createHistoryQueryView(ViewContext context, QueryForm form, BindException errors) + { + UserSchema schema = AuditLogService.getAuditLogSchema(context.getUser(), context.getContainer()); + if (schema != null) + { + QuerySettings settings = new QuerySettings(context, QueryView.DATAREGIONNAME_DEFAULT); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_SCHEMA_NAME), form.getSchemaName()); + filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_QUERY_NAME), form.getQueryName()); + + settings.setBaseFilter(filter); + settings.setQueryName(QUERY_UPDATE_AUDIT_EVENT); + return schema.createView(context, settings, errors); + } + return null; + } + + public static QueryView createDetailsQueryView(ViewContext context, QueryController.QueryDetailsForm form, BindException errors) + { + return createDetailsQueryView(context, form.getSchemaName(), form.getQueryName(), form.getKeyValue(), errors); + } + + public static QueryView createDetailsQueryView(ViewContext context, String schemaName, String queryName, String keyValue, BindException errors) + { + UserSchema schema = AuditLogService.getAuditLogSchema(context.getUser(), context.getContainer()); + if (schema != null) + { + QuerySettings settings = new QuerySettings(context, QueryView.DATAREGIONNAME_DEFAULT); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_SCHEMA_NAME), schemaName); + filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_QUERY_NAME), queryName); + filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_ROW_PK), keyValue); + + settings.setBaseFilter(filter); + settings.setQueryName(QUERY_UPDATE_AUDIT_EVENT); + return schema.createView(context, settings, errors); + } + return null; + } + + public int moveEvents(Container targetContainer, Collection rowPks, String schemaName, String queryName) + { + TableInfo auditTable = createStorageTableInfo(); + SQLFragment sql = new SQLFragment("UPDATE ").append(auditTable) + .append(" SET container = ").appendValue(targetContainer) + .append(" WHERE RowPk "); + auditTable.getSchema().getSqlDialect().appendInClauseSql(sql, rowPks.stream().map(Object::toString).toList()); + sql.append(" AND SchemaName = ").appendValue(schemaName).append(" AND QueryName = ").appendValue(queryName); + return new SqlExecutor(auditTable.getSchema()).execute(sql); + } + + public static class QueryUpdateAuditEvent extends DetailedAuditTypeEvent + { + private String _rowPk; + private String _schemaName; + private String _queryName; + + /** Important for reflection-based instantiation */ + @SuppressWarnings("unused") + public QueryUpdateAuditEvent() + { + super(); + setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); + } + + public QueryUpdateAuditEvent(Container container, String comment) + { + super(QUERY_UPDATE_AUDIT_EVENT, container, comment); + setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); + } + + public String getRowPk() + { + return _rowPk; + } + + public void setRowPk(String rowPk) + { + _rowPk = rowPk; + } + + public String getSchemaName() + { + return _schemaName; + } + + public void setSchemaName(String schemaName) + { + _schemaName = schemaName; + } + + public String getQueryName() + { + return _queryName; + } + + public void setQueryName(String queryName) + { + _queryName = queryName; + } + + @Override + public Map getAuditLogMessageElements() + { + Map elements = new LinkedHashMap<>(); + elements.put("rowPk", getRowPk()); + elements.put("schemaName", getSchemaName()); + elements.put("queryName", getQueryName()); + elements.put("transactionId", getTransactionId()); + elements.put("userComment", getUserComment()); + // N.B. oldRecordMap and newRecordMap are potentially very large (and are not displayed in the default grid view) + elements.putAll(super.getAuditLogMessageElements()); + return elements; + } + } + + public static class QueryUpdateAuditDomainKind extends AbstractAuditDomainKind + { + public static final String NAME = "QueryUpdateAuditDomain"; + public static String NAMESPACE_PREFIX = "Audit-" + NAME; + + private final Set _fields; + + public QueryUpdateAuditDomainKind() + { + super(QUERY_UPDATE_AUDIT_EVENT); + + Set fields = new LinkedHashSet<>(); + fields.add(createPropertyDescriptor(COLUMN_NAME_ROW_PK, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_SCHEMA_NAME, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_QUERY_NAME, PropertyType.STRING)); + fields.add(createOldDataMapPropertyDescriptor()); + fields.add(createNewDataMapPropertyDescriptor()); + fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_ID, PropertyType.BIGINT)); + fields.add(createPropertyDescriptor(COLUMN_NAME_USER_COMMENT, PropertyType.STRING)); + _fields = Collections.unmodifiableSet(fields); + } + + @Override + public Set getProperties() + { + return _fields; + } + + @Override + protected String getNamespacePrefix() + { + return NAMESPACE_PREFIX; + } + + @Override + public String getKindName() + { + return NAME; + } + } +} From ce4bb8a4d8e256b68fa0123e509729d791e5eda7 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Tue, 30 Sep 2025 10:57:51 -0700 Subject: [PATCH 03/11] CRLF --- .../org/labkey/api/exp/list/ListService.java | 130 +++++++++--------- 1 file changed, 65 insertions(+), 65 deletions(-) diff --git a/api/src/org/labkey/api/exp/list/ListService.java b/api/src/org/labkey/api/exp/list/ListService.java index 590c1b402de..9d79a027e38 100644 --- a/api/src/org/labkey/api/exp/list/ListService.java +++ b/api/src/org/labkey/api/exp/list/ListService.java @@ -1,65 +1,65 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.exp.list; - -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.exp.TemplateInfo; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.query.UserSchema; -import org.labkey.api.security.User; -import org.labkey.api.services.ServiceRegistry; -import org.labkey.api.view.ActionURL; -import org.springframework.validation.BindException; - -import java.io.InputStream; -import java.util.Map; - -public interface ListService -{ - static ListService get() - { - return ServiceRegistry.get().getService(ListService.class); - } - - static void setInstance(ListService ls) - { - ServiceRegistry.get().registerService(ListService.class, ls); - } - - Map getLists(Container container); - Map getLists(Container container, @Nullable User user); - Map getLists(Container container, @Nullable User user, boolean checkVisibility); - Map getLists(Container container, @Nullable User user, boolean checkVisibility, boolean includePicklists, boolean includeProjectAndShared); - boolean hasLists(Container container); - boolean hasLists(Container container, boolean includeProjectAndShared); - ListDefinition createList(Container container, String name, ListDefinition.KeyType keyType); - ListDefinition createList(Container container, String name, ListDefinition.KeyType keyType, @Nullable TemplateInfo templateInfo, @Nullable ListDefinition.Category category); - @Nullable ListDefinition getList(Container container, int listId); - @Nullable ListDefinition getList(Container container, String name); - @Nullable ListDefinition getList(Container container, String name, boolean includeProjectAndShared); - ListDefinition getList(Domain domain); - ActionURL getManageListsURL(Container container); - UserSchema getUserSchema(User user, Container container); - - /** Picklists can specify different container filtering configurations depending on the container context */ - @Nullable ContainerFilter getPicklistContainerFilter(Container container, User user, @NotNull ListDefinition list); - - void importListArchive(InputStream is, BindException errors, Container c, User user) throws Exception; -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.exp.list; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.exp.TemplateInfo; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; +import org.labkey.api.services.ServiceRegistry; +import org.labkey.api.view.ActionURL; +import org.springframework.validation.BindException; + +import java.io.InputStream; +import java.util.Map; + +public interface ListService +{ + static ListService get() + { + return ServiceRegistry.get().getService(ListService.class); + } + + static void setInstance(ListService ls) + { + ServiceRegistry.get().registerService(ListService.class, ls); + } + + Map getLists(Container container); + Map getLists(Container container, @Nullable User user); + Map getLists(Container container, @Nullable User user, boolean checkVisibility); + Map getLists(Container container, @Nullable User user, boolean checkVisibility, boolean includePicklists, boolean includeProjectAndShared); + boolean hasLists(Container container); + boolean hasLists(Container container, boolean includeProjectAndShared); + ListDefinition createList(Container container, String name, ListDefinition.KeyType keyType); + ListDefinition createList(Container container, String name, ListDefinition.KeyType keyType, @Nullable TemplateInfo templateInfo, @Nullable ListDefinition.Category category); + @Nullable ListDefinition getList(Container container, int listId); + @Nullable ListDefinition getList(Container container, String name); + @Nullable ListDefinition getList(Container container, String name, boolean includeProjectAndShared); + ListDefinition getList(Domain domain); + ActionURL getManageListsURL(Container container); + UserSchema getUserSchema(User user, Container container); + + /** Picklists can specify different container filtering configurations depending on the container context */ + @Nullable ContainerFilter getPicklistContainerFilter(Container container, User user, @NotNull ListDefinition list); + + void importListArchive(InputStream is, BindException errors, Container c, User user) throws Exception; +} From fd38e6bd9d6a6600f92a088c4c68c32464d8f300 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Tue, 30 Sep 2025 14:51:30 -0700 Subject: [PATCH 04/11] Batch updates --- .../org/labkey/api/data/ContainerManager.java | 25 +++++++- .../org/labkey/api/query/QueryService.java | 2 +- .../experiment/api/ExperimentServiceImpl.java | 8 +-- .../list/model/ListQueryUpdateService.java | 61 +++++++++++-------- .../org/labkey/query/QueryServiceImpl.java | 2 +- .../query/audit/QueryUpdateAuditProvider.java | 18 +++--- 6 files changed, 68 insertions(+), 48 deletions(-) diff --git a/api/src/org/labkey/api/data/ContainerManager.java b/api/src/org/labkey/api/data/ContainerManager.java index bef6d394da8..434dce89b99 100644 --- a/api/src/org/labkey/api/data/ContainerManager.java +++ b/api/src/org/labkey/api/data/ContainerManager.java @@ -2701,6 +2701,11 @@ private static boolean isValidTargetContainer(Container current, Container targe return moveFromProjectToChild || moveFromChildToProject || moveFromChildToSibling; } + public static int updateContainer(TableInfo dataTable, String idField, Collection ids, Container targetContainer) + { + return updateContainer(dataTable, idField, ids, targetContainer, null, false); + } + /** * Updates the container of specified rows in the provided database table. Optionally, the modification timestamp * and the user who made the modification can also be updated if specified. @@ -2714,19 +2719,33 @@ private static boolean isValidTargetContainer(Container current, Container targe * @return The number of rows updated in the table. */ public static int updateContainer(TableInfo dataTable, String idField, Collection ids, Container targetContainer, @Nullable User user, boolean withModified) + { + if (ids == null || ids.isEmpty()) + return 0; + + SimpleFilter filter = new SimpleFilter(); + filter.addInClause(FieldKey.fromParts(idField), ids); + + return updateContainer(dataTable, targetContainer, filter, user, withModified); + } + + public static int updateContainer(TableInfo dataTable, Container targetContainer, @NotNull SimpleFilter filter, @Nullable User user, boolean withModified) { try (DbScope.Transaction transaction = dataTable.getSchema().getScope().ensureTransaction()) { SQLFragment dataUpdate = new SQLFragment("UPDATE ").append(dataTable) .append(" SET container = ").appendValue(targetContainer); + if (withModified) { assert user != null : "User must be specified when updating modified/modifiedBy details."; - dataUpdate.append(", modified = ").appendValue(new Date()); + dataUpdate.append(", modified = CURRENT_TIMESTAMP"); dataUpdate.append(", modifiedby = ").appendValue(user.getUserId()); } - dataUpdate.append(" WHERE ").appendIdentifier(idField); - dataTable.getSchema().getSqlDialect().appendInClauseSql(dataUpdate, ids); + + SQLFragment whereClause = filter.getSQLFragment(dataTable.getSchema().getSqlDialect()); + dataUpdate.append("\n").append(whereClause); + int numUpdated = new SqlExecutor(dataTable.getSchema()).execute(dataUpdate); transaction.commit(); diff --git a/api/src/org/labkey/api/query/QueryService.java b/api/src/org/labkey/api/query/QueryService.java index 922c5af62b9..06b5acf1272 100644 --- a/api/src/org/labkey/api/query/QueryService.java +++ b/api/src/org/labkey/api/query/QueryService.java @@ -462,7 +462,7 @@ public String getDefaultCommentSummary() List getQueryUpdateAuditRecords(User user, Container container, long transactionAuditId, @Nullable ContainerFilter containerFilter); AuditHandler getDefaultAuditHandler(); - int moveAuditEvents(Container targetContainer, List rowPks, String schemaName, String queryName); + int moveAuditEvents(Container targetContainer, Collection rowPks, String schemaName, String queryName); /** * Returns a URL for the audit history for the table. diff --git a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java index 93622d75b78..d2da0ae7575 100644 --- a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java @@ -9561,11 +9561,7 @@ private int updateExpObjectContainers(List lsids, Container targetContai if (lsids == null || lsids.isEmpty()) return 0; - TableInfo objectTable = OntologyManager.getTinfoObject(); - SQLFragment objectUpdate = new SQLFragment("UPDATE ").append(objectTable).append(" SET container = ").appendValue(targetContainer.getEntityId()) - .append(" WHERE objecturi "); - objectTable.getSchema().getSqlDialect().appendInClauseSql(objectUpdate, lsids); - return new SqlExecutor(objectTable.getSchema()).execute(objectUpdate); + return ContainerManager.updateContainer(OntologyManager.getTinfoObject(), "objecturi", lsids, targetContainer); } @Override @@ -9635,7 +9631,7 @@ public Map moveDataClassObjects(Collection d throw errors; // move audit events associated with the sources that are moving - int auditEventCount = QueryService.get().moveAuditEvents(targetContainer, List.of(dataIds), "exp.data", dataClassTable.getName()); + int auditEventCount = QueryService.get().moveAuditEvents(targetContainer, dataIds, "exp.data", dataClassTable.getName()); updateCounts.compute("sourceAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount); // create summary audit entries for the source container only. The message is pretty generic, so having it diff --git a/list/src/org/labkey/list/model/ListQueryUpdateService.java b/list/src/org/labkey/list/model/ListQueryUpdateService.java index 044999a80e8..a42f432a287 100644 --- a/list/src/org/labkey/list/model/ListQueryUpdateService.java +++ b/list/src/org/labkey/list/model/ListQueryUpdateService.java @@ -491,13 +491,13 @@ public Map moveRows( AuditBehaviorType auditBehavior = configParameters != null ? (AuditBehaviorType) configParameters.get(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior) : null; String auditUserComment = configParameters != null ? (String) configParameters.get(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment) : null; User user = getListUser(_user, container); - String listSchemaName = ListSchema.getInstance().getSchemaName(); boolean hasAttachmentProperties = _list.getDomainOrThrow() .getProperties() .stream() .anyMatch(prop -> PropertyType.ATTACHMENT.equals(prop.getPropertyType())); ListAuditProvider listAuditProvider = new ListAuditProvider(); + final int BATCH_SIZE = 5_000; try (DbScope.Transaction tx = getDbTable().getSchema().getScope().ensureTransaction()) { @@ -524,30 +524,47 @@ public Map moveRows( throw new QueryUpdateServiceException(String.format("Failed to retrieve table for list '%s' in folder %s.", _list.getName(), sourceContainer.getPath())); List records = containerRows.get(containerId); - List rowPks = records.stream().map(ListRecord::key).toList(); + int numRecords = records.size(); - Map extraContext = Map.of("targetContainer", targetContainer, "keys", rowPks); - listTable.fireBatchTrigger(sourceContainer, user, TableInfo.TriggerType.MOVE, true, errors, extraContext); - if (errors.hasErrors()) - throw errors; + for (int start = 0; start < numRecords; start += BATCH_SIZE) + { + int end = Math.min(start + BATCH_SIZE, numRecords); + List batch = records.subList(start, end); + List rowPks = batch.stream().map(ListRecord::key).toList(); - listRecordsCount += ContainerManager.updateContainer(getDbTable(), _list.getKeyName(), rowPks, targetContainer, user, true); - if (errors.hasErrors()) - throw errors; + // Before trigger per batch + Map extraContext = Map.of("targetContainer", targetContainer, "keys", rowPks); + listTable.fireBatchTrigger(sourceContainer, user, TableInfo.TriggerType.MOVE, true, errors, extraContext); + if (errors.hasErrors()) + throw errors; - if (hasAttachmentProperties) - { - moveAttachments(user, sourceContainer, targetContainer, records, errors); + listRecordsCount += ContainerManager.updateContainer(getDbTable(), _list.getKeyName(), rowPks, targetContainer, user, true); if (errors.hasErrors()) throw errors; - } - queryAuditEventsMovedCount += QueryService.get().moveAuditEvents(targetContainer, rowPks, listSchemaName, _list.getName()); - listAuditEventsMovedCount += listAuditProvider.moveEvents(targetContainer, records.stream().map(ListRecord::entityId).toList()); + if (hasAttachmentProperties) + { + moveAttachments(user, sourceContainer, targetContainer, batch, errors); + if (errors.hasErrors()) + throw errors; + } + + queryAuditEventsMovedCount += QueryService.get().moveAuditEvents(targetContainer, rowPks, ListQuerySchema.NAME, _list.getName()); + listAuditEventsMovedCount += listAuditProvider.moveEvents(targetContainer, batch.stream().map(ListRecord::entityId).toList()); + + // Detailed audit events per row + if (AuditBehaviorType.DETAILED == listTable.getEffectiveAuditBehavior(auditBehavior)) + listAuditEventsCreatedCount += addDetailedMoveAuditEvents(user, sourceContainer, targetContainer, batch); + + // After trigger per batch + listTable.fireBatchTrigger(sourceContainer, user, TableInfo.TriggerType.MOVE, false, errors, extraContext); + if (errors.hasErrors()) + throw errors; + } // Create a summary audit event for the source container { - String comment = String.format("Moved %s to %s", StringUtilsLabKey.pluralize(records.size(), "row"), targetContainer.getPath()); + String comment = String.format("Moved %s to %s", StringUtilsLabKey.pluralize(numRecords, "row"), targetContainer.getPath()); ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(sourceContainer, comment, _list); event.setUserComment(auditUserComment); listAuditEvents.add(event); @@ -555,20 +572,11 @@ public Map moveRows( // Create a summary audit event for the target container { - String comment = String.format("Moved %s from %s", StringUtilsLabKey.pluralize(records.size(), "row"), sourceContainer.getPath()); + String comment = String.format("Moved %s from %s", StringUtilsLabKey.pluralize(numRecords, "row"), sourceContainer.getPath()); ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(targetContainer, comment, _list); event.setUserComment(auditUserComment); listAuditEvents.add(event); } - - if (AuditBehaviorType.DETAILED == listTable.getEffectiveAuditBehavior(auditBehavior)) - listAuditEventsCreatedCount += addDetailedMoveAuditEvents(user, sourceContainer, targetContainer, records); - - // TODO: Picklist support? - - listTable.fireBatchTrigger(sourceContainer, user, TableInfo.TriggerType.MOVE, false, errors, extraContext); - if (errors.hasErrors()) - throw errors; } if (!listAuditEvents.isEmpty()) @@ -641,7 +649,6 @@ private Map> getListRowsForMoveRows(Container targetConta private void moveAttachments(User user, Container sourceContainer, Container targetContainer, List records, BatchValidationException errors) { - // TODO: Consider subsequent query to determine which rows need to be updated then only update those attachments List parents = new ArrayList<>(); for (ListRecord record : records) parents.add(new ListItemAttachmentParent(record.entityId, sourceContainer)); diff --git a/query/src/org/labkey/query/QueryServiceImpl.java b/query/src/org/labkey/query/QueryServiceImpl.java index d5a2c41447d..1facada39ef 100644 --- a/query/src/org/labkey/query/QueryServiceImpl.java +++ b/query/src/org/labkey/query/QueryServiceImpl.java @@ -3098,7 +3098,7 @@ public void clearEnvironment() } @Override - public int moveAuditEvents(Container targetContainer, List rowPks, String schemaName, String queryName) + public int moveAuditEvents(Container targetContainer, Collection rowPks, String schemaName, String queryName) { QueryUpdateAuditProvider provider = new QueryUpdateAuditProvider(); return provider.moveEvents(targetContainer, rowPks, schemaName, queryName); diff --git a/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java b/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java index 3e2efec92bd..fae57380f4d 100644 --- a/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java +++ b/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java @@ -25,10 +25,9 @@ import org.labkey.api.audit.query.DefaultAuditTypeTable; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; import org.labkey.api.data.MutableColumnInfo; -import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlExecutor; import org.labkey.api.data.TableInfo; import org.labkey.api.exp.PropertyDescriptor; import org.labkey.api.exp.PropertyType; @@ -188,15 +187,14 @@ public static QueryView createDetailsQueryView(ViewContext context, String schem return null; } - public int moveEvents(Container targetContainer, Collection rowPks, String schemaName, String queryName) + public int moveEvents(Container targetContainer, Collection rowPks, String schemaName, String queryName) { - TableInfo auditTable = createStorageTableInfo(); - SQLFragment sql = new SQLFragment("UPDATE ").append(auditTable) - .append(" SET container = ").appendValue(targetContainer) - .append(" WHERE RowPk "); - auditTable.getSchema().getSqlDialect().appendInClauseSql(sql, rowPks.stream().map(Object::toString).toList()); - sql.append(" AND SchemaName = ").appendValue(schemaName).append(" AND QueryName = ").appendValue(queryName); - return new SqlExecutor(auditTable.getSchema()).execute(sql); + SimpleFilter filter = new SimpleFilter(); + filter.addInClause(FieldKey.fromParts(COLUMN_NAME_ROW_PK), rowPks.stream().map(Object::toString).toList()); + filter.addCondition(FieldKey.fromParts(COLUMN_NAME_SCHEMA_NAME), schemaName); + filter.addCondition(FieldKey.fromParts(COLUMN_NAME_QUERY_NAME), queryName); + + return ContainerManager.updateContainer(createStorageTableInfo(), targetContainer, filter, null, false); } public static class QueryUpdateAuditEvent extends DetailedAuditTypeEvent From 1f6f82051078e2c942ff02b45bc2aef54a5863c7 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Thu, 2 Oct 2025 12:59:15 -0700 Subject: [PATCH 05/11] Table.updateContainer --- .../model/AnnouncementManager.java | 2 +- .../api/attachments/AttachmentService.java | 2 +- .../api/audit/AbstractAuditTypeProvider.java | 5 +- .../org/labkey/api/data/ContainerManager.java | 60 +------------ api/src/org/labkey/api/data/Table.java | 86 ++++++++++++++++--- .../org/labkey/api/exp/OntologyManager.java | 2 +- .../attachment/AttachmentServiceImpl.java | 7 +- .../experiment/api/ExperimentServiceImpl.java | 4 +- .../experiment/api/SampleTypeServiceImpl.java | 3 +- .../list/model/ListQueryUpdateService.java | 28 ++++-- .../org/labkey/query/QueryServiceImpl.java | 37 +++++--- .../query/audit/QueryUpdateAuditProvider.java | 3 +- 12 files changed, 141 insertions(+), 98 deletions(-) diff --git a/announcements/src/org/labkey/announcements/model/AnnouncementManager.java b/announcements/src/org/labkey/announcements/model/AnnouncementManager.java index 4ccdffebc80..8fd49a60518 100644 --- a/announcements/src/org/labkey/announcements/model/AnnouncementManager.java +++ b/announcements/src/org/labkey/announcements/model/AnnouncementManager.java @@ -696,7 +696,7 @@ public static AnnouncementModel updateAnnouncement(User user, AnnouncementModel public static int updateContainer(List discussionSrcIds, Container targetContainer, User user) { - return ContainerManager.updateContainer(_comm.getTableInfoAnnouncements(), "discussionSrcIdentifier", discussionSrcIds, targetContainer, user, false); + return Table.updateContainer(_comm.getTableInfoAnnouncements(), "discussionSrcIdentifier", discussionSrcIds, targetContainer, user, false); } diff --git a/api/src/org/labkey/api/attachments/AttachmentService.java b/api/src/org/labkey/api/attachments/AttachmentService.java index 91e937f8007..538d25674f9 100644 --- a/api/src/org/labkey/api/attachments/AttachmentService.java +++ b/api/src/org/labkey/api/attachments/AttachmentService.java @@ -91,7 +91,7 @@ static AttachmentService get() void copyAttachment(AttachmentParent parent, Attachment a, String newName, User auditUser) throws IOException; - void moveAttachments(Container newContainer, List parents, User auditUser) throws IOException; + int moveAttachments(Container newContainer, List parents, User auditUser) throws IOException; @NotNull List getAttachmentFiles(AttachmentParent parent, Collection attachments) throws IOException; diff --git a/api/src/org/labkey/api/audit/AbstractAuditTypeProvider.java b/api/src/org/labkey/api/audit/AbstractAuditTypeProvider.java index ccab9d16d9b..da95c90b575 100644 --- a/api/src/org/labkey/api/audit/AbstractAuditTypeProvider.java +++ b/api/src/org/labkey/api/audit/AbstractAuditTypeProvider.java @@ -30,8 +30,7 @@ import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.DbScope; import org.labkey.api.data.MutableColumnInfo; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.Table; import org.labkey.api.data.TableInfo; import org.labkey.api.dataiterator.DataIterator; import org.labkey.api.dataiterator.ExistingRecordDataIterator; @@ -395,6 +394,6 @@ else if (value instanceof Date date) public int moveEvents(Container targetContainer, String idColumnName, Collection ids) { - return ContainerManager.updateContainer(createStorageTableInfo(), idColumnName, ids, targetContainer, null, false); + return Table.updateContainer(createStorageTableInfo(), idColumnName, ids, targetContainer, null, false); } } diff --git a/api/src/org/labkey/api/data/ContainerManager.java b/api/src/org/labkey/api/data/ContainerManager.java index 434dce89b99..b160aeb4c27 100644 --- a/api/src/org/labkey/api/data/ContainerManager.java +++ b/api/src/org/labkey/api/data/ContainerManager.java @@ -43,6 +43,7 @@ import org.labkey.api.audit.provider.ContainerAuditProvider; import org.labkey.api.cache.Cache; import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.collections.ConcurrentHashSet; import org.labkey.api.collections.IntHashMap; @@ -103,7 +104,6 @@ import org.labkey.api.view.ViewContext; import org.labkey.api.writer.MemoryVirtualFile; import org.labkey.folder.xml.FolderDocument; -import org.labkey.remoteapi.collections.CaseInsensitiveHashMap; import org.springframework.validation.BindException; import org.springframework.validation.Errors; @@ -2684,7 +2684,7 @@ private static Container getContainerForIdOrPath(String targetContainer) } // targetContainer must be in the same app project at this time - // i.e. child of current project, project of current child, sibling within project + // i.e., child of the current project, project of the current child, descendants within the project private static boolean isValidTargetContainer(Container current, Container target) { if (current.isRoot() || target.isRoot()) @@ -2696,61 +2696,9 @@ private static boolean isValidTargetContainer(Container current, Container targe boolean moveFromProjectToChild = current.isProject() && target.getParent().equals(current); boolean moveFromChildToProject = !current.isProject() && current.getParent().isProject() && current.getParent().equals(target); - boolean moveFromChildToSibling = !current.isProject() && current.getParent().isProject() && current.getParent().equals(target.getParent()); + boolean moveFromChildToChildWithinProject = !current.isProject() && !target.isProject() && current.getProject() != null && current.getProject().equals(target.getProject()); - return moveFromProjectToChild || moveFromChildToProject || moveFromChildToSibling; - } - - public static int updateContainer(TableInfo dataTable, String idField, Collection ids, Container targetContainer) - { - return updateContainer(dataTable, idField, ids, targetContainer, null, false); - } - - /** - * Updates the container of specified rows in the provided database table. Optionally, the modification timestamp - * and the user who made the modification can also be updated if specified. - * - * @param dataTable The table where the container update should be applied. - * @param idField The name of the identifier field used to locate the rows to update. - * @param ids A collection of identifier values specifying the rows to be updated. - * @param targetContainer The target container to set for the specified rows. - * @param user The user performing the update. If null, modified/modifiedBy details are not updated. - * @param withModified If true, updates the modified timestamp and the user who made the modification. - * @return The number of rows updated in the table. - */ - public static int updateContainer(TableInfo dataTable, String idField, Collection ids, Container targetContainer, @Nullable User user, boolean withModified) - { - if (ids == null || ids.isEmpty()) - return 0; - - SimpleFilter filter = new SimpleFilter(); - filter.addInClause(FieldKey.fromParts(idField), ids); - - return updateContainer(dataTable, targetContainer, filter, user, withModified); - } - - public static int updateContainer(TableInfo dataTable, Container targetContainer, @NotNull SimpleFilter filter, @Nullable User user, boolean withModified) - { - try (DbScope.Transaction transaction = dataTable.getSchema().getScope().ensureTransaction()) - { - SQLFragment dataUpdate = new SQLFragment("UPDATE ").append(dataTable) - .append(" SET container = ").appendValue(targetContainer); - - if (withModified) - { - assert user != null : "User must be specified when updating modified/modifiedBy details."; - dataUpdate.append(", modified = CURRENT_TIMESTAMP"); - dataUpdate.append(", modifiedby = ").appendValue(user.getUserId()); - } - - SQLFragment whereClause = filter.getSQLFragment(dataTable.getSchema().getSqlDialect()); - dataUpdate.append("\n").append(whereClause); - - int numUpdated = new SqlExecutor(dataTable.getSchema()).execute(dataUpdate); - transaction.commit(); - - return numUpdated; - } + return moveFromProjectToChild || moveFromChildToProject || moveFromChildToChildWithinProject; } /** diff --git a/api/src/org/labkey/api/data/Table.java b/api/src/org/labkey/api/data/Table.java index f14716c0b14..e4612ee972c 100644 --- a/api/src/org/labkey/api/data/Table.java +++ b/api/src/org/labkey/api/data/Table.java @@ -963,7 +963,7 @@ public static K update(@Nullable User user, TableInfo table, K fieldsIn, Obj /* NOTE this does not enforce that keyColumn is an appropriately unique column! */ public static K update(@Nullable User user, TableInfo table, K fieldsIn, @Nullable ColumnInfo keyColumn, Object pkVals, @Nullable Filter filter, Level level) { - assert (table.getTableType() != DatabaseTableType.NOT_IN_DB): (table.getName() + " is not in the physical database."); + assert assertInDb(table); assert null != pkVals; // _executeTriggers(table, previous, fields); @@ -1128,6 +1128,65 @@ else if (pkVals instanceof Map) return (fieldsIn instanceof Map && !(fieldsIn instanceof BoundMap)) ? (K)fields : fieldsIn; } + /** + * Updates the container of specified rows in the provided database table. Optionally, the modification timestamp + * and the user who made the modification can also be updated if specified. + * + * @param table The table where the container update should be applied. + * @param idField The name of the identifier field used to locate the rows to update. + * @param ids A collection of identifier values specifying the rows to be updated. + * @param targetContainer The target container to set for the specified rows. + * @param user The user performing the update. If null, modified/modifiedBy details are not updated. + * @param withModified If true, updates the modified timestamp and the user who made the modification. + * @return The number of rows updated in the table. + */ + public static int updateContainer(TableInfo table, String idField, Collection ids, Container targetContainer, @Nullable User user, boolean withModified) + { + assert assertInDb(table); + ColumnInfo idColumn = table.getColumn(idField); + if (idColumn == null) + throw new IllegalArgumentException("Table " + table.getSchema().getName() + "." + table.getName() + " has no column named '" + idField + "'."); + + if (ids == null || ids.isEmpty()) + return 0; + + SimpleFilter filter = new SimpleFilter(); + filter.addInClause(idColumn.getFieldKey(), ids); + + return updateContainer(table, targetContainer, filter, user, withModified); + } + + public static int updateContainer(TableInfo table, Container targetContainer, @NotNull SimpleFilter filter, @Nullable User user, boolean withModified) + { + assert assertInDb(table); + SQLFragment dataUpdate = new SQLFragment("UPDATE ").append(table) + .append(" SET container = ").appendValue(targetContainer); + + if (withModified) + { + assert user != null : "User must be specified when updating modified/modifiedBy details."; + ColumnInfo colModified = table.getColumn(MODIFIED_COLUMN_NAME); + if (null != colModified) + { + dataUpdate.append(", ").appendIdentifier(colModified.getSelectIdentifier()) + .append(" = ") + .appendValue(new java.sql.Timestamp(System.currentTimeMillis())); + } + + ColumnInfo colModifiedBy = table.getColumn(MODIFIED_BY_COLUMN_NAME); + if (null != colModifiedBy) + { + dataUpdate.append(", ").appendIdentifier(colModifiedBy.getSelectIdentifier()) + .append(" = ") + .appendValue(user.getUserId()); + } + } + + SQLFragment whereClause = filter.getSQLFragment(table.getSqlDialect(), null, createMetaDataNameMap(table)); + dataUpdate.append("\n").append(whereClause); + + return new SqlExecutor(table.getSchema()).execute(dataUpdate); + } public static void delete(TableInfo table, Object rowId) { @@ -1154,17 +1213,16 @@ public static void delete(TableInfo table, Object rowId) public static int delete(TableInfo table) { - assert (table.getTableType() != DatabaseTableType.NOT_IN_DB): (table.getName() + " is not in the physical database."); + assert assertInDb(table); SqlExecutor sqlExecutor = new SqlExecutor(table.getSchema()); + return sqlExecutor.execute("DELETE FROM " + table.getSelectName()); } public static int delete(TableInfo table, Filter filter) { - assert (table.getTableType() != DatabaseTableType.NOT_IN_DB): (table.getName() + " is not in the physical database."); - + assert assertInDb(table); SQLFragment where = filter.getSQLFragment(table.getSqlDialect(), null, createMetaDataNameMap(table)); - SQLFragment deleteSQL = new SQLFragment("DELETE FROM ").append(table).append("\n\t").append(where); return new SqlExecutor(table.getSchema()).execute(deleteSQL); @@ -1172,7 +1230,7 @@ public static int delete(TableInfo table, Filter filter) public static void truncate(TableInfo table) { - assert (table.getTableType() != DatabaseTableType.NOT_IN_DB): (table.getName() + " is not in the physical database."); + assert assertInDb(table); SqlExecutor sqlExecutor = new SqlExecutor(table.getSchema()); sqlExecutor.execute(table.getSqlDialect().getTruncateSql(table.getSelectName())); } @@ -1569,7 +1627,7 @@ static public Map createColumnMap(@Nullable TableInfo tabl * Create a map that can be passed into Filter.getSQLFragment() that create a SQL fragment using getMetaDataName() instead of * getAlias(). */ - static private Map createMetaDataNameMap(TableInfo table) + private static Map createMetaDataNameMap(TableInfo table) { Map ret = new HashMap<>(); for (var column : table.getColumns()) @@ -1581,7 +1639,6 @@ static private Map createMetaDataNameMap(TableInfo table) return ret; } - public static boolean checkAllColumns(TableInfo table, Collection columns, String prefix) { return checkAllColumns(table, columns, prefix, false); @@ -1591,7 +1648,6 @@ public static boolean checkAllColumns(TableInfo table, Collection co { int bad = 0; -// Map mapFK = new HashMap<>(columns.size()*2); Map mapAlias = new HashMap<>(columns.size()*2); ColumnInfo prev; @@ -1599,8 +1655,6 @@ public static boolean checkAllColumns(TableInfo table, Collection co { if (!checkColumn(table, column, prefix)) bad++; -// if (enforceUnique && null != (prev=mapFK.put(column.getFieldKey(), column)) && prev != column) -// bad++; if (enforceUnique && !(column instanceof AliasedColumn) && null != (prev = mapAlias.put(column.getAlias().getId(), column)) && prev != column) { _log.warn(prefix + ": Column " + column + " from table: " + column.getParentTable() + " is mapped to the same alias (" + column.getAlias().getId() + ") as column " + prev + " from table: " + prev.getParentTable()); @@ -1621,7 +1675,6 @@ public static boolean checkAllColumns(TableInfo table, Collection co return 0 == bad; } - public static boolean checkColumn(TableInfo table, ColumnInfo column, String prefix) { if (column.getParentTable() != table) @@ -1635,8 +1688,7 @@ public static boolean checkColumn(TableInfo table, ColumnInfo column, String pre } } - - public static ParameterMapStatement deleteStatement(Connection conn, TableInfo tableDelete /*, Set columns */) throws SQLException + public static ParameterMapStatement deleteStatement(Connection conn, TableInfo tableDelete) throws SQLException { if (!(tableDelete instanceof UpdateableTableInfo updatable)) throw new IllegalArgumentException(); @@ -1732,6 +1784,12 @@ public static ParameterMapStatement deleteStatement(Connection conn, TableInfo t return new ParameterMapStatement(tableDelete.getSchema().getScope(), conn, sqlfDelete, updatable.remapSchemaColumns()); } + private static boolean assertInDb(TableInfo table) + { + if (table.getTableType() == DatabaseTableType.NOT_IN_DB) + throw new AssertionError("Table " + table.getSchema().getName() + "." + table.getName() + " is not in the physical database."); + return true; + } public static class TestDataIterator extends AbstractDataIterator { diff --git a/api/src/org/labkey/api/exp/OntologyManager.java b/api/src/org/labkey/api/exp/OntologyManager.java index 2fab041f709..26d1b6d43ec 100644 --- a/api/src/org/labkey/api/exp/OntologyManager.java +++ b/api/src/org/labkey/api/exp/OntologyManager.java @@ -927,7 +927,7 @@ public static void updateObjectPropertyOrder(User user, Container container, Str */ public static int updateContainer(Container targetContainer, User user, @NotNull String objectLSID) { - return ContainerManager.updateContainer(getTinfoObject(), "objectURI", List.of(objectLSID), targetContainer, user, false); + return Table.updateContainer(getTinfoObject(), "objectURI", List.of(objectLSID), targetContainer, user, false); } /** diff --git a/core/src/org/labkey/core/attachment/AttachmentServiceImpl.java b/core/src/org/labkey/core/attachment/AttachmentServiceImpl.java index 4f29e16b95e..b6e72f8779f 100644 --- a/core/src/org/labkey/core/attachment/AttachmentServiceImpl.java +++ b/core/src/org/labkey/core/attachment/AttachmentServiceImpl.java @@ -545,14 +545,17 @@ public void copyAttachment(AttachmentParent parent, Attachment a, String newName } @Override - public void moveAttachments(Container newContainer, List parents, User auditUser) throws IOException + public int moveAttachments(Container newContainer, List parents, User auditUser) throws IOException { + int totalRowsChanged = 0; + for (AttachmentParent parent : parents) { checkSecurityPolicy(auditUser, parent); int rowsChanged = new SqlExecutor(coreTables().getSchema()).execute(sqlMove(parent, newContainer)); if (rowsChanged > 0) { + totalRowsChanged += rowsChanged; List atts = getAttachments(parent); String filename; for (Attachment att : atts) @@ -575,6 +578,8 @@ public void moveAttachments(Container newContainer, List paren AttachmentCache.removeAttachments(parent); } } + + return totalRowsChanged; } /** may return fewer AttachmentFile than Attachment, if there have been deletions */ diff --git a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java index d2da0ae7575..5d2e5f94876 100644 --- a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java @@ -9561,7 +9561,7 @@ private int updateExpObjectContainers(List lsids, Container targetContai if (lsids == null || lsids.isEmpty()) return 0; - return ContainerManager.updateContainer(OntologyManager.getTinfoObject(), "objecturi", lsids, targetContainer); + return Table.updateContainer(OntologyManager.getTinfoObject(), "objecturi", lsids, targetContainer, null, false); } @Override @@ -9610,7 +9610,7 @@ public Map moveDataClassObjects(Collection d TableInfo dataClassTable = schema.getTable(dataClass.getName()); // update exp.data.container - int updateCount = ContainerManager.updateContainer(getTinfoData(), "rowId", dataIds, targetContainer, user, true); + int updateCount = Table.updateContainer(getTinfoData(), "rowId", dataIds, targetContainer, user, true); updateCounts.put("sources", updateCounts.get("sources") + updateCount); // update for exp.object.container diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java index 13257230cfa..9109ecd0e27 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java @@ -59,6 +59,7 @@ import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.SqlExecutor; import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.Table; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.data.dialect.SqlDialect; @@ -1896,7 +1897,7 @@ public Map moveSamples(Collection sample List sampleIds = typeSamples.stream().map(ExpMaterial::getRowId).toList(); // update for exp.material.container - updateCounts.put("samples", updateCounts.get("samples") + ContainerManager.updateContainer(getTinfoMaterial(), "rowid", sampleIds, targetContainer, user, true)); + updateCounts.put("samples", updateCounts.get("samples") + Table.updateContainer(getTinfoMaterial(), "rowid", sampleIds, targetContainer, user, true)); // update for exp.object.container expService.updateExpObjectContainers(getTinfoMaterial(), sampleIds, targetContainer); diff --git a/list/src/org/labkey/list/model/ListQueryUpdateService.java b/list/src/org/labkey/list/model/ListQueryUpdateService.java index a42f432a287..1a2bb5b1e23 100644 --- a/list/src/org/labkey/list/model/ListQueryUpdateService.java +++ b/list/src/org/labkey/list/model/ListQueryUpdateService.java @@ -36,6 +36,7 @@ import org.labkey.api.data.RuntimeSQLException; import org.labkey.api.data.Selector.ForEachBatchBlock; import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Table; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.dataiterator.DataIteratorBuilder; @@ -47,6 +48,7 @@ import org.labkey.api.exp.list.ListDefinition; import org.labkey.api.exp.list.ListImportProgress; import org.labkey.api.exp.list.ListItem; +import org.labkey.api.exp.list.ListService; import org.labkey.api.exp.property.Domain; import org.labkey.api.exp.property.DomainProperty; import org.labkey.api.exp.property.IPropertyValidator; @@ -476,17 +478,25 @@ public Map moveRows( @Nullable Map extraScriptContext ) throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException { + var resolvedList = ListService.get().getList(targetContainer, _list.getName(), true); + if (resolvedList == null) + { + errors.addRowError(new ValidationException(String.format("List '%s' is not accessible from folder %s.", _list.getName(), targetContainer.getPath()))); + throw errors; + } + Map> containerRows = getListRowsForMoveRows(targetContainer, rows, errors); if (errors.hasErrors()) throw errors; + int fileAttachmentsMovedCount = 0; int listAuditEventsCreatedCount = 0; int listAuditEventsMovedCount = 0; int listRecordsCount = 0; int queryAuditEventsMovedCount = 0; if (containerRows.isEmpty()) - return moveRowsCounts(listAuditEventsCreatedCount, listAuditEventsMovedCount, listRecordsCount, queryAuditEventsMovedCount); + return moveRowsCounts(fileAttachmentsMovedCount, listAuditEventsCreatedCount, listAuditEventsMovedCount, listRecordsCount, queryAuditEventsMovedCount); AuditBehaviorType auditBehavior = configParameters != null ? (AuditBehaviorType) configParameters.get(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior) : null; String auditUserComment = configParameters != null ? (String) configParameters.get(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment) : null; @@ -538,13 +548,13 @@ public Map moveRows( if (errors.hasErrors()) throw errors; - listRecordsCount += ContainerManager.updateContainer(getDbTable(), _list.getKeyName(), rowPks, targetContainer, user, true); + listRecordsCount += Table.updateContainer(getDbTable(), _list.getKeyName(), rowPks, targetContainer, user, true); if (errors.hasErrors()) throw errors; if (hasAttachmentProperties) { - moveAttachments(user, sourceContainer, targetContainer, batch, errors); + fileAttachmentsMovedCount += moveAttachments(user, sourceContainer, targetContainer, batch, errors); if (errors.hasErrors()) throw errors; } @@ -592,12 +602,13 @@ public Map moveRows( SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "moveEntities", "list"); } - return moveRowsCounts(listAuditEventsCreatedCount, listAuditEventsMovedCount, listRecordsCount, queryAuditEventsMovedCount); + return moveRowsCounts(fileAttachmentsMovedCount, listAuditEventsCreatedCount, listAuditEventsMovedCount, listRecordsCount, queryAuditEventsMovedCount); } - private Map moveRowsCounts(int listAuditEventsCreated, int listAuditEventsMoved, int listRecords, int queryAuditEventsMoved) + private Map moveRowsCounts(int fileAttachmentsMovedCount, int listAuditEventsCreated, int listAuditEventsMoved, int listRecords, int queryAuditEventsMoved) { return Map.of( + "fileAttachmentsMoved", fileAttachmentsMovedCount, "listAuditEventsCreated", listAuditEventsCreated, "listAuditEventsMoved", listAuditEventsMoved, "listRecords", listRecords, @@ -647,20 +658,23 @@ private Map> getListRowsForMoveRows(Container targetConta return containerRows; } - private void moveAttachments(User user, Container sourceContainer, Container targetContainer, List records, BatchValidationException errors) + private int moveAttachments(User user, Container sourceContainer, Container targetContainer, List records, BatchValidationException errors) { List parents = new ArrayList<>(); for (ListRecord record : records) parents.add(new ListItemAttachmentParent(record.entityId, sourceContainer)); + int count = 0; try { - AttachmentService.get().moveAttachments(targetContainer, parents, user); + count = AttachmentService.get().moveAttachments(targetContainer, parents, user); } catch (IOException e) { errors.addRowError(new ValidationException("Failed to move attachments when moving list rows. Error: " + e.getMessage())); } + + return count; } private int addDetailedMoveAuditEvents(User user, Container sourceContainer, Container targetContainer, List records) diff --git a/query/src/org/labkey/query/QueryServiceImpl.java b/query/src/org/labkey/query/QueryServiceImpl.java index 1facada39ef..1f76ca1522f 100644 --- a/query/src/org/labkey/query/QueryServiceImpl.java +++ b/query/src/org/labkey/query/QueryServiceImpl.java @@ -3135,21 +3135,38 @@ private QueryUpdateAuditProvider.QueryUpdateAuditEvent createAuditRecord(Contain FieldKey rowPk = tinfo.getAuditRowPk(); if (rowPk != null) { - String rowPkStr = rowPk.toString(); - Object pk = null; - if (row != null && row.containsKey(rowPkStr)) - { - pk = row.get(rowPkStr); - } - if (pk == null && existingRow != null && existingRow.containsKey(rowPkStr)) - { - pk = existingRow.get(rowPkStr); - } + Object pk = resolvePkValue(rowPk, row); + if (pk == null) + pk = resolvePkValue(rowPk, existingRow); + + assert pk != null : String.format( + "Unable to determine \"RowPk\" value for query audit record for table \"%s\".\"%s\". Expected value for \"%s\" to be available.", + tinfo.getSchema().getName(), + tinfo.getName(), + rowPk.getName() + ); + if (pk != null) event.setRowPk(String.valueOf(pk)); } + return event; } + + private static @Nullable Object resolvePkValue(@NotNull FieldKey fieldKey, @Nullable Map row) + { + Object pk = null; + if (row != null) + { + String rowPkStr = fieldKey.toString(); + if (row.containsKey(rowPkStr)) + pk = row.get(rowPkStr); + if (pk == null && row.containsKey(fieldKey.getName())) + pk = row.get(fieldKey.getName()); + } + + return pk; + } }; } diff --git a/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java b/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java index fae57380f4d..a00220d7cfe 100644 --- a/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java +++ b/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java @@ -28,6 +28,7 @@ import org.labkey.api.data.ContainerManager; import org.labkey.api.data.MutableColumnInfo; import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Table; import org.labkey.api.data.TableInfo; import org.labkey.api.exp.PropertyDescriptor; import org.labkey.api.exp.PropertyType; @@ -194,7 +195,7 @@ public int moveEvents(Container targetContainer, Collection rowPks, String sc filter.addCondition(FieldKey.fromParts(COLUMN_NAME_SCHEMA_NAME), schemaName); filter.addCondition(FieldKey.fromParts(COLUMN_NAME_QUERY_NAME), queryName); - return ContainerManager.updateContainer(createStorageTableInfo(), targetContainer, filter, null, false); + return Table.updateContainer(createStorageTableInfo(), targetContainer, filter, null, false); } public static class QueryUpdateAuditEvent extends DetailedAuditTypeEvent From 90a46fcc4a58a793142c4e3b56ec80c543c58638 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Thu, 2 Oct 2025 13:14:43 -0700 Subject: [PATCH 06/11] No row is OK --- query/src/org/labkey/query/QueryServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/query/src/org/labkey/query/QueryServiceImpl.java b/query/src/org/labkey/query/QueryServiceImpl.java index 1f76ca1522f..d7bdaa1ad65 100644 --- a/query/src/org/labkey/query/QueryServiceImpl.java +++ b/query/src/org/labkey/query/QueryServiceImpl.java @@ -3133,7 +3133,7 @@ private QueryUpdateAuditProvider.QueryUpdateAuditEvent createAuditRecord(Contain event.setQueryName(tinfo.getPublicName()); FieldKey rowPk = tinfo.getAuditRowPk(); - if (rowPk != null) + if (rowPk != null && (row != null || existingRow != null)) { Object pk = resolvePkValue(rowPk, row); if (pk == null) From e48a70ae62c8801be4a5792de1fc48c1cd4bc793 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Thu, 2 Oct 2025 15:29:39 -0700 Subject: [PATCH 07/11] Remove audit assertion --- query/src/org/labkey/query/QueryServiceImpl.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/query/src/org/labkey/query/QueryServiceImpl.java b/query/src/org/labkey/query/QueryServiceImpl.java index d7bdaa1ad65..009b239df58 100644 --- a/query/src/org/labkey/query/QueryServiceImpl.java +++ b/query/src/org/labkey/query/QueryServiceImpl.java @@ -3139,13 +3139,6 @@ private QueryUpdateAuditProvider.QueryUpdateAuditEvent createAuditRecord(Contain if (pk == null) pk = resolvePkValue(rowPk, existingRow); - assert pk != null : String.format( - "Unable to determine \"RowPk\" value for query audit record for table \"%s\".\"%s\". Expected value for \"%s\" to be available.", - tinfo.getSchema().getName(), - tinfo.getName(), - rowPk.getName() - ); - if (pk != null) event.setRowPk(String.valueOf(pk)); } From a5fc4c3937d99df783f926b7756de10e598cf389 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Fri, 3 Oct 2025 12:19:34 -0700 Subject: [PATCH 08/11] Review feedback --- .../org/labkey/api/data/ContainerManager.java | 16 +- api/src/org/labkey/api/data/Table.java | 14 +- .../org/labkey/api/query/QueryService.java | 9 + .../api/assay/AssayResultDomainKind.java | 385 +++++++++--------- .../labkey/api/assay/AssayResultTable.java | 34 +- .../org/labkey/list/model/ListItemImpl.java | 3 - .../list/model/ListQueryUpdateService.java | 42 +- .../query/audit/QueryUpdateAuditProvider.java | 1 - 8 files changed, 269 insertions(+), 235 deletions(-) diff --git a/api/src/org/labkey/api/data/ContainerManager.java b/api/src/org/labkey/api/data/ContainerManager.java index b160aeb4c27..af7d9098e1a 100644 --- a/api/src/org/labkey/api/data/ContainerManager.java +++ b/api/src/org/labkey/api/data/ContainerManager.java @@ -2683,8 +2683,7 @@ private static Container getContainerForIdOrPath(String targetContainer) return c; } - // targetContainer must be in the same app project at this time - // i.e., child of the current project, project of the current child, descendants within the project + // Current and target containers must be within the same project tree at this time private static boolean isValidTargetContainer(Container current, Container target) { if (current.isRoot() || target.isRoot()) @@ -2694,11 +2693,16 @@ private static boolean isValidTargetContainer(Container current, Container targe if (current.equals(target)) return true; - boolean moveFromProjectToChild = current.isProject() && target.getParent().equals(current); - boolean moveFromChildToProject = !current.isProject() && current.getParent().isProject() && current.getParent().equals(target); - boolean moveFromChildToChildWithinProject = !current.isProject() && !target.isProject() && current.getProject() != null && current.getProject().equals(target.getProject()); + // from project to descendant + if (current.isProject()) + return target.isDescendant(current); - return moveFromProjectToChild || moveFromChildToProject || moveFromChildToChildWithinProject; + // from descendant to project + if (target.isProject()) + return current.isDescendant(target); + + // from descendant to descendant + return current.getProject() != null && current.getProject().equals(target.getProject()); } /** diff --git a/api/src/org/labkey/api/data/Table.java b/api/src/org/labkey/api/data/Table.java index e4612ee972c..8fe3ddc7fef 100644 --- a/api/src/org/labkey/api/data/Table.java +++ b/api/src/org/labkey/api/data/Table.java @@ -21,7 +21,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.Logger; -import org.apache.poi.ss.formula.functions.T; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.junit.Assert; @@ -37,6 +36,7 @@ import org.labkey.api.dataiterator.DataIteratorContext; import org.labkey.api.dataiterator.Pump; import org.labkey.api.dataiterator.SimpleTranslator; +import org.labkey.api.dataiterator.SimpleTranslator.SpecialColumn; import org.labkey.api.dataiterator.TableInsertDataIteratorBuilder; import org.labkey.api.di.DataIntegrationService; import org.labkey.api.exceptions.OptimisticConflictException; @@ -102,12 +102,12 @@ public class Table // Makes long parameter lists easier to read public static final int NO_OFFSET = 0; - public static final String ENTITY_ID_COLUMN_NAME = "EntityId"; - public static final String OWNER_COLUMN_NAME = "Owner"; - public static final String CREATED_BY_COLUMN_NAME = "CreatedBy"; - public static final String CREATED_COLUMN_NAME = "Created"; - public static final String MODIFIED_BY_COLUMN_NAME = "ModifiedBy"; - public static final String MODIFIED_COLUMN_NAME = "Modified"; + private static final String ENTITY_ID_COLUMN_NAME = SpecialColumn.EntityId.name(); + private static final String OWNER_COLUMN_NAME = "Owner"; + private static final String CREATED_BY_COLUMN_NAME = SpecialColumn.CreatedBy.name(); + private static final String CREATED_COLUMN_NAME = SpecialColumn.Created.name(); + private static final String MODIFIED_BY_COLUMN_NAME = SpecialColumn.ModifiedBy.name(); + private static final String MODIFIED_COLUMN_NAME = SpecialColumn.Modified.name(); /** Columns that are magically populated as part of an insert or update operation */ public static final Set AUTOPOPULATED_COLUMN_NAMES = CaseInsensitiveHashSet.of( diff --git a/api/src/org/labkey/api/query/QueryService.java b/api/src/org/labkey/api/query/QueryService.java index 06b5acf1272..844b9cb3db6 100644 --- a/api/src/org/labkey/api/query/QueryService.java +++ b/api/src/org/labkey/api/query/QueryService.java @@ -462,6 +462,15 @@ public String getDefaultCommentSummary() List getQueryUpdateAuditRecords(User user, Container container, long transactionAuditId, @Nullable ContainerFilter containerFilter); AuditHandler getDefaultAuditHandler(); + /** + * Moves audit events associated with the specific rows, identified by primary key, to the target container. + * + * @param targetContainer The container to which audit events will be moved. + * @param rowPks A collection of primary key values identifying the rows whose audit events should be moved. + * @param schemaName The schema name of the table. + * @param queryName The query (table) name. + * @return The number of audit events moved. + */ int moveAuditEvents(Container targetContainer, Collection rowPks, String schemaName, String queryName); /** diff --git a/assay/api-src/org/labkey/api/assay/AssayResultDomainKind.java b/assay/api-src/org/labkey/api/assay/AssayResultDomainKind.java index d6e4000a671..da71127f4f8 100644 --- a/assay/api-src/org/labkey/api/assay/AssayResultDomainKind.java +++ b/assay/api-src/org/labkey/api/assay/AssayResultDomainKind.java @@ -1,194 +1,191 @@ -/* - * Copyright (c) 2010-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.assay; - -import org.jetbrains.annotations.NotNull; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.data.Container; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.PropertyStorageSpec; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainUtil; -import org.labkey.api.query.FieldKey; -import org.labkey.api.security.User; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static org.labkey.api.assay.AssayFileWriter.DIR_NAME; -import static org.labkey.api.data.Table.CREATED_BY_COLUMN_NAME; -import static org.labkey.api.data.Table.CREATED_COLUMN_NAME; -import static org.labkey.api.data.Table.MODIFIED_BY_COLUMN_NAME; -import static org.labkey.api.data.Table.MODIFIED_COLUMN_NAME; - -public class AssayResultDomainKind extends AssayDomainKind -{ - private static final Set RESERVED_NAMES; - static - { - RESERVED_NAMES = new CaseInsensitiveHashSet(getAssayReservedPropertyNames()); - RESERVED_NAMES.addAll(DomainUtil.getNamesAndLabels(List.of("Run", "DataId"))); - } - - public enum Column - { - Plate, - Replicate, - ReplicateLsid, - State, - WellLocation, - WellLsid; - - public FieldKey fieldKey() - { - return FieldKey.fromParts(name()); - } - } - - public AssayResultDomainKind() - { - super(ExpProtocol.ASSAY_DOMAIN_DATA); - } - - @Override - public String getKindName() - { - return "Assay Results"; - } - - @Override - public Set getBaseProperties(Domain domain) - { - PropertyStorageSpec dataIdSpec = new PropertyStorageSpec(AbstractTsvAssayProvider.DATA_ID_COLUMN_NAME, JdbcType.INTEGER); - dataIdSpec.setNullable(false); - - PropertyStorageSpec rowIdSpec = new PropertyStorageSpec(AbstractTsvAssayProvider.ROW_ID_COLUMN_NAME, JdbcType.INTEGER); - rowIdSpec.setAutoIncrement(true); - rowIdSpec.setPrimaryKey(true); - - PropertyStorageSpec createdSpec = new PropertyStorageSpec(CREATED_COLUMN_NAME, JdbcType.TIMESTAMP); - PropertyStorageSpec createdBySpec = new PropertyStorageSpec(CREATED_BY_COLUMN_NAME, JdbcType.INTEGER); - PropertyStorageSpec modifiedSpec = new PropertyStorageSpec(MODIFIED_COLUMN_NAME, JdbcType.TIMESTAMP); - PropertyStorageSpec modifiedBySpec = new PropertyStorageSpec(MODIFIED_BY_COLUMN_NAME, JdbcType.INTEGER); - - return PageFlowUtil.set(rowIdSpec, dataIdSpec, createdSpec, createdBySpec, modifiedSpec, modifiedBySpec); - } - - @Override - public Set getPropertyIndices(Domain domain) - { - return PageFlowUtil.set(new PropertyStorageSpec.Index(false, AbstractTsvAssayProvider.DATA_ID_COLUMN_NAME)); - } - - @Override - public Set getPropertyForeignKeys(Container container) - { - return new HashSet<>(Arrays.asList( - new PropertyStorageSpec.ForeignKey(CREATED_BY_COLUMN_NAME, "core", "users", "userid", null, false), - new PropertyStorageSpec.ForeignKey(MODIFIED_BY_COLUMN_NAME, "core", "users", "userid", null, false) - )); - } - - @Override - public DbScope getScope() - { - return getSchema().getScope(); - } - - @Override - public String getStorageSchemaName() - { - return AbstractTsvAssayProvider.ASSAY_SCHEMA_NAME; - } - - public DbSchema getSchema() - { - return DbSchema.get(getStorageSchemaName(), getSchemaType()); - } - - @Override - public @NotNull Set getReservedPropertyNames(Domain domain, User user) - { - return RESERVED_NAMES; - } - - @Override - public Set getMandatoryPropertyNames(Domain domain) - { - Set mandatoryNames = super.getMandatoryPropertyNames(domain); - - Pair pair = findProviderAndProtocol(domain); - if (pair != null) - { - AssayProvider provider = pair.first; - ExpProtocol protocol = pair.second; - if (provider != null && protocol != null) - { - if (provider.isPlateMetadataEnabled(protocol)) - { - mandatoryNames.add(Column.Plate.name()); - mandatoryNames.add(Column.WellLocation.name()); - mandatoryNames.add(Column.WellLsid.name()); - mandatoryNames.add(Column.ReplicateLsid.name()); - mandatoryNames.add(Column.State.name()); - } - } - } - - return mandatoryNames; - } - - @Override - public boolean allowCalculatedFields() - { - return true; - } - - @Override - public void deletePropertyDescriptor(Domain domain, User user, PropertyDescriptor pd) - { - super.deletePropertyDescriptor(domain, user, pd); - - // SQL Server does not allow for multiple foreign keys to the same table to utilize ON DELETE CASCADE as it may - // cause cycles or multiple cascade paths. The solution is to only ON DELETE CASCADE for one foreign key and - // clean up upon delete of the property for other changes. See the "CREATE TABLE assay.FilterCriteria" - // statement in assay schema upgrade scripts. - if (!OntologyManager.getSqlDialect().isSqlServer()) - return; - - Pair pair = findProviderAndProtocol(domain); - if (pair == null) - return; - - pair.first.removeFilterCriteriaForProperty(pd); - } - - @Override - public String getDomainFileDirectory() - { - return DIR_NAME; - } -} +/* + * Copyright (c) 2010-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.assay; + +import org.jetbrains.annotations.NotNull; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.data.Container; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.dataiterator.SimpleTranslator.SpecialColumn; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainUtil; +import org.labkey.api.query.FieldKey; +import org.labkey.api.security.User; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.labkey.api.assay.AssayFileWriter.DIR_NAME; + +public class AssayResultDomainKind extends AssayDomainKind +{ + private static final Set RESERVED_NAMES; + static + { + RESERVED_NAMES = new CaseInsensitiveHashSet(getAssayReservedPropertyNames()); + RESERVED_NAMES.addAll(DomainUtil.getNamesAndLabels(List.of("Run", "DataId"))); + } + + public enum Column + { + Plate, + Replicate, + ReplicateLsid, + State, + WellLocation, + WellLsid; + + public FieldKey fieldKey() + { + return FieldKey.fromParts(name()); + } + } + + public AssayResultDomainKind() + { + super(ExpProtocol.ASSAY_DOMAIN_DATA); + } + + @Override + public String getKindName() + { + return "Assay Results"; + } + + @Override + public Set getBaseProperties(Domain domain) + { + PropertyStorageSpec dataIdSpec = new PropertyStorageSpec(AbstractTsvAssayProvider.DATA_ID_COLUMN_NAME, JdbcType.INTEGER); + dataIdSpec.setNullable(false); + + PropertyStorageSpec rowIdSpec = new PropertyStorageSpec(AbstractTsvAssayProvider.ROW_ID_COLUMN_NAME, JdbcType.INTEGER); + rowIdSpec.setAutoIncrement(true); + rowIdSpec.setPrimaryKey(true); + + PropertyStorageSpec createdSpec = new PropertyStorageSpec(SpecialColumn.Created.name(), JdbcType.TIMESTAMP); + PropertyStorageSpec createdBySpec = new PropertyStorageSpec(SpecialColumn.CreatedBy.name(), JdbcType.INTEGER); + PropertyStorageSpec modifiedSpec = new PropertyStorageSpec(SpecialColumn.Modified.name(), JdbcType.TIMESTAMP); + PropertyStorageSpec modifiedBySpec = new PropertyStorageSpec(SpecialColumn.ModifiedBy.name(), JdbcType.INTEGER); + + return PageFlowUtil.set(rowIdSpec, dataIdSpec, createdSpec, createdBySpec, modifiedSpec, modifiedBySpec); + } + + @Override + public Set getPropertyIndices(Domain domain) + { + return PageFlowUtil.set(new PropertyStorageSpec.Index(false, AbstractTsvAssayProvider.DATA_ID_COLUMN_NAME)); + } + + @Override + public Set getPropertyForeignKeys(Container container) + { + return new HashSet<>(Arrays.asList( + new PropertyStorageSpec.ForeignKey(SpecialColumn.CreatedBy.name(), "core", "users", "userid", null, false), + new PropertyStorageSpec.ForeignKey(SpecialColumn.ModifiedBy.name(), "core", "users", "userid", null, false) + )); + } + + @Override + public DbScope getScope() + { + return getSchema().getScope(); + } + + @Override + public String getStorageSchemaName() + { + return AbstractTsvAssayProvider.ASSAY_SCHEMA_NAME; + } + + public DbSchema getSchema() + { + return DbSchema.get(getStorageSchemaName(), getSchemaType()); + } + + @Override + public @NotNull Set getReservedPropertyNames(Domain domain, User user) + { + return RESERVED_NAMES; + } + + @Override + public Set getMandatoryPropertyNames(Domain domain) + { + Set mandatoryNames = super.getMandatoryPropertyNames(domain); + + Pair pair = findProviderAndProtocol(domain); + if (pair != null) + { + AssayProvider provider = pair.first; + ExpProtocol protocol = pair.second; + if (provider != null && protocol != null) + { + if (provider.isPlateMetadataEnabled(protocol)) + { + mandatoryNames.add(Column.Plate.name()); + mandatoryNames.add(Column.WellLocation.name()); + mandatoryNames.add(Column.WellLsid.name()); + mandatoryNames.add(Column.ReplicateLsid.name()); + mandatoryNames.add(Column.State.name()); + } + } + } + + return mandatoryNames; + } + + @Override + public boolean allowCalculatedFields() + { + return true; + } + + @Override + public void deletePropertyDescriptor(Domain domain, User user, PropertyDescriptor pd) + { + super.deletePropertyDescriptor(domain, user, pd); + + // SQL Server does not allow for multiple foreign keys to the same table to utilize ON DELETE CASCADE as it may + // cause cycles or multiple cascade paths. The solution is to only ON DELETE CASCADE for one foreign key and + // clean up upon delete of the property for other changes. See the "CREATE TABLE assay.FilterCriteria" + // statement in assay schema upgrade scripts. + if (!OntologyManager.getSqlDialect().isSqlServer()) + return; + + Pair pair = findProviderAndProtocol(domain); + if (pair == null) + return; + + pair.first.removeFilterCriteriaForProperty(pd); + } + + @Override + public String getDomainFileDirectory() + { + return DIR_NAME; + } +} diff --git a/assay/api-src/org/labkey/api/assay/AssayResultTable.java b/assay/api-src/org/labkey/api/assay/AssayResultTable.java index 5e4dc295054..35711099c37 100644 --- a/assay/api-src/org/labkey/api/assay/AssayResultTable.java +++ b/assay/api-src/org/labkey/api/assay/AssayResultTable.java @@ -38,6 +38,7 @@ import org.labkey.api.data.dialect.SqlDialect; import org.labkey.api.dataiterator.DataIteratorBuilder; import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.SimpleTranslator.SpecialColumn; import org.labkey.api.dataiterator.TableInsertDataIteratorBuilder; import org.labkey.api.exp.MvColumn; import org.labkey.api.exp.PropertyColumn; @@ -79,11 +80,6 @@ import java.util.Map; import java.util.Set; -import static org.labkey.api.data.Table.CREATED_BY_COLUMN_NAME; -import static org.labkey.api.data.Table.CREATED_COLUMN_NAME; -import static org.labkey.api.data.Table.MODIFIED_BY_COLUMN_NAME; -import static org.labkey.api.data.Table.MODIFIED_COLUMN_NAME; - public class AssayResultTable extends FilteredTable implements UpdateableTableInfo { protected final ExpProtocol _protocol; @@ -331,12 +327,16 @@ public static BaseColumnInfo createRowExpressionLsidColumn(FilteredTable CREATED_MODIFIED_COLUMN_NAMES = CaseInsensitiveHashSet.of( + SpecialColumn.Created.name(), + SpecialColumn.CreatedBy.name(), + SpecialColumn.Modified.name(), + SpecialColumn.ModifiedBy.name() + ); + private boolean isCreatedModifiedCol(String colName) { - return CREATED_COLUMN_NAME.equalsIgnoreCase(colName) || - CREATED_BY_COLUMN_NAME.equalsIgnoreCase(colName) || - MODIFIED_COLUMN_NAME.equalsIgnoreCase(colName) || - MODIFIED_BY_COLUMN_NAME.equalsIgnoreCase(colName); + return CREATED_MODIFIED_COLUMN_NAMES.contains(colName); } // Expensive render-time fetching of all ontology properties attached to the object row @@ -437,10 +437,10 @@ protected boolean shouldIncludeCreatedModified(Set selectedColumns) if (null == selectedColumns) // select all return true; - return selectedColumns.contains(new FieldKey(null, CREATED_COLUMN_NAME)) || - selectedColumns.contains(new FieldKey(null, CREATED_BY_COLUMN_NAME)) || - selectedColumns.contains(new FieldKey(null, MODIFIED_COLUMN_NAME)) || - selectedColumns.contains(new FieldKey(null, MODIFIED_BY_COLUMN_NAME)); + return selectedColumns.contains(new FieldKey(null, SpecialColumn.Created.name())) || + selectedColumns.contains(new FieldKey(null, SpecialColumn.CreatedBy.name())) || + selectedColumns.contains(new FieldKey(null, SpecialColumn.Modified.name())) || + selectedColumns.contains(new FieldKey(null, SpecialColumn.ModifiedBy.name())); } @NotNull @@ -468,10 +468,10 @@ public SQLFragment getFromSQLExpanded(String alias, Set selectedColumn // without any updates to the results rows, the created and modified date of the result should match the created date of the run // use run.created/by as default result modified/by String runSelectCol = propertyColumn.getName(); - if (MODIFIED_COLUMN_NAME.equalsIgnoreCase(runSelectCol)) - runSelectCol = CREATED_COLUMN_NAME; - else if (MODIFIED_BY_COLUMN_NAME.equalsIgnoreCase(runSelectCol)) - runSelectCol = CREATED_BY_COLUMN_NAME; + if (SpecialColumn.Modified.name().equalsIgnoreCase(runSelectCol)) + runSelectCol = SpecialColumn.Created.name(); + else if (SpecialColumn.ModifiedBy.name().equalsIgnoreCase(runSelectCol)) + runSelectCol = SpecialColumn.CreatedBy.name(); coalescedCol.append(runSelectCol); coalescedCol.append(") AS "); diff --git a/list/src/org/labkey/list/model/ListItemImpl.java b/list/src/org/labkey/list/model/ListItemImpl.java index b8eb65807df..1a9c4bae87c 100644 --- a/list/src/org/labkey/list/model/ListItemImpl.java +++ b/list/src/org/labkey/list/model/ListItemImpl.java @@ -16,8 +16,6 @@ package org.labkey.list.model; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.labkey.api.exp.ObjectProperty; import org.labkey.api.exp.OntologyManager; import org.labkey.api.exp.OntologyObject; @@ -36,7 +34,6 @@ public class ListItemImpl implements ListItem ListItm _itm; Map _properties; Map _oldProperties; - private static final Logger _log = LogManager.getLogger(ListItemImpl.class); public ListItemImpl(ListDefinitionImpl list, ListItm item) { diff --git a/list/src/org/labkey/list/model/ListQueryUpdateService.java b/list/src/org/labkey/list/model/ListQueryUpdateService.java index 1a2bb5b1e23..44787baad7b 100644 --- a/list/src/org/labkey/list/model/ListQueryUpdateService.java +++ b/list/src/org/labkey/list/model/ListQueryUpdateService.java @@ -30,6 +30,7 @@ import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.DbScope; import org.labkey.api.data.LookupResolutionType; @@ -70,6 +71,7 @@ import org.labkey.api.security.User; import org.labkey.api.security.permissions.DeletePermission; import org.labkey.api.security.permissions.MoveEntitiesPermission; +import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.security.permissions.UpdatePermission; import org.labkey.api.security.roles.EditorRole; import org.labkey.api.usageMetrics.SimpleMetricsService; @@ -87,8 +89,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import static org.labkey.api.util.IntegerUtils.isIntegral; @@ -478,14 +482,15 @@ public Map moveRows( @Nullable Map extraScriptContext ) throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException { - var resolvedList = ListService.get().getList(targetContainer, _list.getName(), true); - if (resolvedList == null) + // Ensure the list is in scope for the target container + if (null == ListService.get().getList(targetContainer, _list.getName(), true)) { errors.addRowError(new ValidationException(String.format("List '%s' is not accessible from folder %s.", _list.getName(), targetContainer.getPath()))); throw errors; } - Map> containerRows = getListRowsForMoveRows(targetContainer, rows, errors); + User user = getListUser(_user, container); + Map> containerRows = getListRowsForMoveRows(container, user, targetContainer, rows, errors); if (errors.hasErrors()) throw errors; @@ -500,7 +505,6 @@ public Map moveRows( AuditBehaviorType auditBehavior = configParameters != null ? (AuditBehaviorType) configParameters.get(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior) : null; String auditUserComment = configParameters != null ? (String) configParameters.get(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment) : null; - User user = getListUser(_user, container); boolean hasAttachmentProperties = _list.getDomainOrThrow() .getProperties() .stream() @@ -508,10 +512,11 @@ public Map moveRows( ListAuditProvider listAuditProvider = new ListAuditProvider(); final int BATCH_SIZE = 5_000; + boolean isAuditEnabled = auditBehavior != null && AuditBehaviorType.NONE != auditBehavior; try (DbScope.Transaction tx = getDbTable().getSchema().getScope().ensureTransaction()) { - if (auditBehavior != null && AuditBehaviorType.NONE != auditBehavior && tx.getAuditEvent() == null) + if (isAuditEnabled && tx.getAuditEvent() == null) { TransactionAuditProvider.TransactionAuditEvent auditEvent = createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); auditEvent.updateCommentRowCount(containerRows.values().stream().mapToInt(List::size).sum()); @@ -573,6 +578,7 @@ public Map moveRows( } // Create a summary audit event for the source container + if (isAuditEnabled) { String comment = String.format("Moved %s to %s", StringUtilsLabKey.pluralize(numRecords, "row"), targetContainer.getPath()); ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(sourceContainer, comment, _list); @@ -581,6 +587,7 @@ public Map moveRows( } // Create a summary audit event for the target container + if (isAuditEnabled) { String comment = String.format("Moved %s from %s", StringUtilsLabKey.pluralize(numRecords, "row"), sourceContainer.getPath()); ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(targetContainer, comment, _list); @@ -616,7 +623,13 @@ private Map moveRowsCounts(int fileAttachmentsMovedCount, int li ); } - private Map> getListRowsForMoveRows(Container targetContainer, List> rows, BatchValidationException errors) + private Map> getListRowsForMoveRows( + Container container, + User user, + Container targetContainer, + List> rows, + BatchValidationException errors + ) throws QueryUpdateServiceException { if (rows.isEmpty()) return Collections.emptyMap(); @@ -640,12 +653,27 @@ private Map> getListRowsForMoveRows(Container targetConta filter.addInClause(fieldKey, keys); filter.addCondition(FieldKey.fromParts("Container"), targetContainer.getId(), CompareType.NEQ); + // Request all rows without a container filter so that rows are more easily resolved across the list scope. + // Read permissions are subsequently checked upon loading a row. + TableInfo table = _list.getTable(user, container, ContainerFilter.getUnsafeEverythingFilter()); + if (table == null) + throw new QueryUpdateServiceException(String.format("Failed to resolve table for list %s in %s", _list.getName(), container.getPath())); + Map> containerRows = new HashMap<>(); - try (var result = new TableSelector(getQueryTable(), PageFlowUtil.set(keyName, "Container", "EntityId"), filter, null).getResults()) + try (var result = new TableSelector(table, PageFlowUtil.set(keyName, "Container", "EntityId"), filter, null).getResults()) { while (result.next()) { GUID containerId = new GUID(result.getString("Container")); + if (!containerRows.containsKey(containerId)) + { + var c = ContainerManager.getForId(containerId); + if (c == null) + throw new QueryUpdateServiceException(String.format("Failed to resolve container for row in list %s in %s.", _list.getName(), container.getPath())); + else if (!c.hasPermission(user, ReadPermission.class)) + throw new UnauthorizedException("You do not have permission to read list rows in all source containers."); + } + containerRows.computeIfAbsent(containerId, k -> new ArrayList<>()); containerRows.get(containerId).add(new ListRecord(result.getObject(fieldKey), result.getString("EntityId"))); } diff --git a/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java b/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java index a00220d7cfe..76f970a6e05 100644 --- a/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java +++ b/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java @@ -25,7 +25,6 @@ import org.labkey.api.audit.query.DefaultAuditTypeTable; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; import org.labkey.api.data.MutableColumnInfo; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.Table; From 0cfffc4b5c99706a5a5ceb00d01256701223fc42 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Fri, 3 Oct 2025 12:51:46 -0700 Subject: [PATCH 09/11] More feedback --- api/src/org/labkey/api/data/Table.java | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/api/src/org/labkey/api/data/Table.java b/api/src/org/labkey/api/data/Table.java index 8fe3ddc7fef..ac80f5b39a5 100644 --- a/api/src/org/labkey/api/data/Table.java +++ b/api/src/org/labkey/api/data/Table.java @@ -819,7 +819,7 @@ else if (propName.endsWith("id")) */ public static K insert(@Nullable User user, TableInfo table, K fieldsIn) { - assert (table.getTableType() != DatabaseTableType.NOT_IN_DB): ("Table " + table.getSchema().getName() + "." + table.getName() + " is not in the physical database."); + assert assertInDb(table); // _executeTriggers(table, fields); @@ -1145,7 +1145,7 @@ public static int updateContainer(TableInfo table, String idField, Collection assert assertInDb(table); ColumnInfo idColumn = table.getColumn(idField); if (idColumn == null) - throw new IllegalArgumentException("Table " + table.getSchema().getName() + "." + table.getName() + " has no column named '" + idField + "'."); + throw new IllegalArgumentException("Table " + fullTableName(table) + " has no column named '" + idField + "'."); if (ids == null || ids.isEmpty()) return 0; @@ -1159,8 +1159,14 @@ public static int updateContainer(TableInfo table, String idField, Collection public static int updateContainer(TableInfo table, Container targetContainer, @NotNull SimpleFilter filter, @Nullable User user, boolean withModified) { assert assertInDb(table); + ColumnInfo containerColumn = table.getColumn(SpecialColumn.Container.name()); + if (containerColumn == null) + throw new IllegalArgumentException("Table " + fullTableName(table) + " has no column named '" + SpecialColumn.Container.name() + "'."); + SQLFragment dataUpdate = new SQLFragment("UPDATE ").append(table) - .append(" SET container = ").appendValue(targetContainer); + .append(" SET ").appendIdentifier(containerColumn.getSelectIdentifier()) + .append(" = ") + .appendValue(targetContainer); if (withModified) { @@ -1787,10 +1793,15 @@ public static ParameterMapStatement deleteStatement(Connection conn, TableInfo t private static boolean assertInDb(TableInfo table) { if (table.getTableType() == DatabaseTableType.NOT_IN_DB) - throw new AssertionError("Table " + table.getSchema().getName() + "." + table.getName() + " is not in the physical database."); + throw new AssertionError("Table " + fullTableName(table) + " is not in the physical database."); return true; } + private static String fullTableName(TableInfo table) + { + return table.getSchema().getName() + "." + table.getName(); + } + public static class TestDataIterator extends AbstractDataIterator { private final String guid = GUID.makeGUID(); From 1918e2185b139a25bee06a82bcde1f34fee6d200 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Fri, 3 Oct 2025 14:09:04 -0700 Subject: [PATCH 10/11] Fix audit list detail links/resolution --- .../list/controllers/ListController.java | 63 +++++++++++++++---- .../labkey/list/model/ListAuditProvider.java | 2 +- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/list/src/org/labkey/list/controllers/ListController.java b/list/src/org/labkey/list/controllers/ListController.java index 474d2ff170a..78a641ffec7 100644 --- a/list/src/org/labkey/list/controllers/ListController.java +++ b/list/src/org/labkey/list/controllers/ListController.java @@ -146,11 +146,6 @@ import java.util.Set; import java.util.TreeSet; -/** - * User: adam - * Date: Dec 30, 2007 - * Time: 12:44:30 PM - */ public class ListController extends SpringActionController { private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(ListController.class, ClearDefaultValuesAction.class); @@ -848,27 +843,71 @@ private String getUrlParam(Enum param) return form.getReturnUrl(); } + public static class ListItemDetailsForm + { + private Integer _listId; + private String _name; + private Integer _rowId; + + public Integer getListId() + { + return _listId; + } + + public void setListId(Integer listId) + { + _listId = listId; + } + + public String getName() + { + return _name; + } + + public void setName(String name) + { + _name = name; + } + + public Integer getRowId() + { + return _rowId; + } + + public void setRowId(Integer rowId) + { + _rowId = rowId; + } + } + @RequiresPermission(ReadPermission.class) - public class ListItemDetailsAction extends SimpleViewAction + public class ListItemDetailsAction extends SimpleViewAction { private ListDefinition _list; @Override - public ModelAndView getView(Object o, BindException errors) + public ModelAndView getView(ListItemDetailsForm form, BindException errors) { - int id = NumberUtils.toInt((String)getViewContext().get("rowId")); - int listId = NumberUtils.toInt((String)getViewContext().get("listId")); - _list = ListService.get().getList(getContainer(), listId); + String listName = form.getName(); + if (listName != null) + _list = ListService.get().getList(getContainer(), listName, true); + if (_list == null) { - return HtmlView.of("This list is no longer available."); + int listId = form.getListId(); + if (listId > 0) + _list = ListService.get().getList(getContainer(), listId); } + if (_list == null) + return HtmlView.of("This list is no longer available."); + String comment = null; String oldRecord = null; String newRecord = null; - ListAuditProvider.ListAuditEvent event = AuditLogService.get().getAuditEvent(getUser(), ListManager.LIST_AUDIT_EVENT, id); + int eventRowId = form.getRowId(); + ListAuditProvider.ListAuditEvent event = AuditLogService.get().getAuditEvent(getUser(), ListManager.LIST_AUDIT_EVENT, eventRowId); if (event != null) { diff --git a/list/src/org/labkey/list/model/ListAuditProvider.java b/list/src/org/labkey/list/model/ListAuditProvider.java index 028f99770f9..e91414869f8 100644 --- a/list/src/org/labkey/list/model/ListAuditProvider.java +++ b/list/src/org/labkey/list/model/ListAuditProvider.java @@ -104,7 +104,7 @@ protected void initColumn(MutableColumnInfo col) appendValueMapColumns(table, null, true); // Render a details URL only for rows that have a listItemEntityId - DetailsURL url = DetailsURL.fromString("list/listItemDetails.view?listId=${listId}&entityId=${listItemEntityId}&rowId=${rowId}", null, StringExpressionFactory.AbstractStringExpression.NullValueBehavior.NullResult); + DetailsURL url = DetailsURL.fromString("list/listItemDetails.view?listId=${listId}&name=${listName}&entityId=${listItemEntityId}&rowId=${rowId}", null, StringExpressionFactory.AbstractStringExpression.NullValueBehavior.NullResult); table.setDetailsURL(url); return table; From 62f4ab98b8d7a3b681b7a4bc8e6ab7c0a3953516 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Fri, 3 Oct 2025 14:13:27 -0700 Subject: [PATCH 11/11] CLRF --- .../api/assay/AssayResultDomainKind.java | 382 +++++++++--------- 1 file changed, 191 insertions(+), 191 deletions(-) diff --git a/assay/api-src/org/labkey/api/assay/AssayResultDomainKind.java b/assay/api-src/org/labkey/api/assay/AssayResultDomainKind.java index da71127f4f8..707d9644d66 100644 --- a/assay/api-src/org/labkey/api/assay/AssayResultDomainKind.java +++ b/assay/api-src/org/labkey/api/assay/AssayResultDomainKind.java @@ -1,191 +1,191 @@ -/* - * Copyright (c) 2010-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.assay; - -import org.jetbrains.annotations.NotNull; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.data.Container; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.PropertyStorageSpec; -import org.labkey.api.dataiterator.SimpleTranslator.SpecialColumn; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainUtil; -import org.labkey.api.query.FieldKey; -import org.labkey.api.security.User; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static org.labkey.api.assay.AssayFileWriter.DIR_NAME; - -public class AssayResultDomainKind extends AssayDomainKind -{ - private static final Set RESERVED_NAMES; - static - { - RESERVED_NAMES = new CaseInsensitiveHashSet(getAssayReservedPropertyNames()); - RESERVED_NAMES.addAll(DomainUtil.getNamesAndLabels(List.of("Run", "DataId"))); - } - - public enum Column - { - Plate, - Replicate, - ReplicateLsid, - State, - WellLocation, - WellLsid; - - public FieldKey fieldKey() - { - return FieldKey.fromParts(name()); - } - } - - public AssayResultDomainKind() - { - super(ExpProtocol.ASSAY_DOMAIN_DATA); - } - - @Override - public String getKindName() - { - return "Assay Results"; - } - - @Override - public Set getBaseProperties(Domain domain) - { - PropertyStorageSpec dataIdSpec = new PropertyStorageSpec(AbstractTsvAssayProvider.DATA_ID_COLUMN_NAME, JdbcType.INTEGER); - dataIdSpec.setNullable(false); - - PropertyStorageSpec rowIdSpec = new PropertyStorageSpec(AbstractTsvAssayProvider.ROW_ID_COLUMN_NAME, JdbcType.INTEGER); - rowIdSpec.setAutoIncrement(true); - rowIdSpec.setPrimaryKey(true); - - PropertyStorageSpec createdSpec = new PropertyStorageSpec(SpecialColumn.Created.name(), JdbcType.TIMESTAMP); - PropertyStorageSpec createdBySpec = new PropertyStorageSpec(SpecialColumn.CreatedBy.name(), JdbcType.INTEGER); - PropertyStorageSpec modifiedSpec = new PropertyStorageSpec(SpecialColumn.Modified.name(), JdbcType.TIMESTAMP); - PropertyStorageSpec modifiedBySpec = new PropertyStorageSpec(SpecialColumn.ModifiedBy.name(), JdbcType.INTEGER); - - return PageFlowUtil.set(rowIdSpec, dataIdSpec, createdSpec, createdBySpec, modifiedSpec, modifiedBySpec); - } - - @Override - public Set getPropertyIndices(Domain domain) - { - return PageFlowUtil.set(new PropertyStorageSpec.Index(false, AbstractTsvAssayProvider.DATA_ID_COLUMN_NAME)); - } - - @Override - public Set getPropertyForeignKeys(Container container) - { - return new HashSet<>(Arrays.asList( - new PropertyStorageSpec.ForeignKey(SpecialColumn.CreatedBy.name(), "core", "users", "userid", null, false), - new PropertyStorageSpec.ForeignKey(SpecialColumn.ModifiedBy.name(), "core", "users", "userid", null, false) - )); - } - - @Override - public DbScope getScope() - { - return getSchema().getScope(); - } - - @Override - public String getStorageSchemaName() - { - return AbstractTsvAssayProvider.ASSAY_SCHEMA_NAME; - } - - public DbSchema getSchema() - { - return DbSchema.get(getStorageSchemaName(), getSchemaType()); - } - - @Override - public @NotNull Set getReservedPropertyNames(Domain domain, User user) - { - return RESERVED_NAMES; - } - - @Override - public Set getMandatoryPropertyNames(Domain domain) - { - Set mandatoryNames = super.getMandatoryPropertyNames(domain); - - Pair pair = findProviderAndProtocol(domain); - if (pair != null) - { - AssayProvider provider = pair.first; - ExpProtocol protocol = pair.second; - if (provider != null && protocol != null) - { - if (provider.isPlateMetadataEnabled(protocol)) - { - mandatoryNames.add(Column.Plate.name()); - mandatoryNames.add(Column.WellLocation.name()); - mandatoryNames.add(Column.WellLsid.name()); - mandatoryNames.add(Column.ReplicateLsid.name()); - mandatoryNames.add(Column.State.name()); - } - } - } - - return mandatoryNames; - } - - @Override - public boolean allowCalculatedFields() - { - return true; - } - - @Override - public void deletePropertyDescriptor(Domain domain, User user, PropertyDescriptor pd) - { - super.deletePropertyDescriptor(domain, user, pd); - - // SQL Server does not allow for multiple foreign keys to the same table to utilize ON DELETE CASCADE as it may - // cause cycles or multiple cascade paths. The solution is to only ON DELETE CASCADE for one foreign key and - // clean up upon delete of the property for other changes. See the "CREATE TABLE assay.FilterCriteria" - // statement in assay schema upgrade scripts. - if (!OntologyManager.getSqlDialect().isSqlServer()) - return; - - Pair pair = findProviderAndProtocol(domain); - if (pair == null) - return; - - pair.first.removeFilterCriteriaForProperty(pd); - } - - @Override - public String getDomainFileDirectory() - { - return DIR_NAME; - } -} +/* + * Copyright (c) 2010-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.assay; + +import org.jetbrains.annotations.NotNull; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.data.Container; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.dataiterator.SimpleTranslator.SpecialColumn; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainUtil; +import org.labkey.api.query.FieldKey; +import org.labkey.api.security.User; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.labkey.api.assay.AssayFileWriter.DIR_NAME; + +public class AssayResultDomainKind extends AssayDomainKind +{ + private static final Set RESERVED_NAMES; + static + { + RESERVED_NAMES = new CaseInsensitiveHashSet(getAssayReservedPropertyNames()); + RESERVED_NAMES.addAll(DomainUtil.getNamesAndLabels(List.of("Run", "DataId"))); + } + + public enum Column + { + Plate, + Replicate, + ReplicateLsid, + State, + WellLocation, + WellLsid; + + public FieldKey fieldKey() + { + return FieldKey.fromParts(name()); + } + } + + public AssayResultDomainKind() + { + super(ExpProtocol.ASSAY_DOMAIN_DATA); + } + + @Override + public String getKindName() + { + return "Assay Results"; + } + + @Override + public Set getBaseProperties(Domain domain) + { + PropertyStorageSpec dataIdSpec = new PropertyStorageSpec(AbstractTsvAssayProvider.DATA_ID_COLUMN_NAME, JdbcType.INTEGER); + dataIdSpec.setNullable(false); + + PropertyStorageSpec rowIdSpec = new PropertyStorageSpec(AbstractTsvAssayProvider.ROW_ID_COLUMN_NAME, JdbcType.INTEGER); + rowIdSpec.setAutoIncrement(true); + rowIdSpec.setPrimaryKey(true); + + PropertyStorageSpec createdSpec = new PropertyStorageSpec(SpecialColumn.Created.name(), JdbcType.TIMESTAMP); + PropertyStorageSpec createdBySpec = new PropertyStorageSpec(SpecialColumn.CreatedBy.name(), JdbcType.INTEGER); + PropertyStorageSpec modifiedSpec = new PropertyStorageSpec(SpecialColumn.Modified.name(), JdbcType.TIMESTAMP); + PropertyStorageSpec modifiedBySpec = new PropertyStorageSpec(SpecialColumn.ModifiedBy.name(), JdbcType.INTEGER); + + return PageFlowUtil.set(rowIdSpec, dataIdSpec, createdSpec, createdBySpec, modifiedSpec, modifiedBySpec); + } + + @Override + public Set getPropertyIndices(Domain domain) + { + return PageFlowUtil.set(new PropertyStorageSpec.Index(false, AbstractTsvAssayProvider.DATA_ID_COLUMN_NAME)); + } + + @Override + public Set getPropertyForeignKeys(Container container) + { + return new HashSet<>(Arrays.asList( + new PropertyStorageSpec.ForeignKey(SpecialColumn.CreatedBy.name(), "core", "users", "userid", null, false), + new PropertyStorageSpec.ForeignKey(SpecialColumn.ModifiedBy.name(), "core", "users", "userid", null, false) + )); + } + + @Override + public DbScope getScope() + { + return getSchema().getScope(); + } + + @Override + public String getStorageSchemaName() + { + return AbstractTsvAssayProvider.ASSAY_SCHEMA_NAME; + } + + public DbSchema getSchema() + { + return DbSchema.get(getStorageSchemaName(), getSchemaType()); + } + + @Override + public @NotNull Set getReservedPropertyNames(Domain domain, User user) + { + return RESERVED_NAMES; + } + + @Override + public Set getMandatoryPropertyNames(Domain domain) + { + Set mandatoryNames = super.getMandatoryPropertyNames(domain); + + Pair pair = findProviderAndProtocol(domain); + if (pair != null) + { + AssayProvider provider = pair.first; + ExpProtocol protocol = pair.second; + if (provider != null && protocol != null) + { + if (provider.isPlateMetadataEnabled(protocol)) + { + mandatoryNames.add(Column.Plate.name()); + mandatoryNames.add(Column.WellLocation.name()); + mandatoryNames.add(Column.WellLsid.name()); + mandatoryNames.add(Column.ReplicateLsid.name()); + mandatoryNames.add(Column.State.name()); + } + } + } + + return mandatoryNames; + } + + @Override + public boolean allowCalculatedFields() + { + return true; + } + + @Override + public void deletePropertyDescriptor(Domain domain, User user, PropertyDescriptor pd) + { + super.deletePropertyDescriptor(domain, user, pd); + + // SQL Server does not allow for multiple foreign keys to the same table to utilize ON DELETE CASCADE as it may + // cause cycles or multiple cascade paths. The solution is to only ON DELETE CASCADE for one foreign key and + // clean up upon delete of the property for other changes. See the "CREATE TABLE assay.FilterCriteria" + // statement in assay schema upgrade scripts. + if (!OntologyManager.getSqlDialect().isSqlServer()) + return; + + Pair pair = findProviderAndProtocol(domain); + if (pair == null) + return; + + pair.first.removeFilterCriteriaForProperty(pd); + } + + @Override + public String getDomainFileDirectory() + { + return DIR_NAME; + } +}