From 2776fd82d2dbfa0053585b348df6d0fac5797acf Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Mon, 6 Oct 2025 12:07:05 -0700 Subject: [PATCH 1/3] Issue 48864: use CURRENT_TIMESTAMP instead of system time --- .../org/labkey/api/data/BaseColumnInfo.java | 2 +- api/src/org/labkey/api/data/SQLFragment.java | 2708 ++++++------ .../org/labkey/api/data/StatementUtils.java | 3833 +++++++++-------- api/src/org/labkey/api/data/Table.java | 42 +- .../api/dataiterator/SimpleTranslator.java | 46 +- audit/src/org/labkey/audit/AuditLogImpl.java | 643 ++- .../org/labkey/audit/model/LogManager.java | 583 ++- 7 files changed, 3922 insertions(+), 3935 deletions(-) diff --git a/api/src/org/labkey/api/data/BaseColumnInfo.java b/api/src/org/labkey/api/data/BaseColumnInfo.java index 6990d288da8..f1a1d08a00a 100644 --- a/api/src/org/labkey/api/data/BaseColumnInfo.java +++ b/api/src/org/labkey/api/data/BaseColumnInfo.java @@ -904,7 +904,7 @@ public SQLFragment getVersionUpdateExpression() { if (JdbcType.TIMESTAMP == getJdbcType()) { - return new SQLFragment("CURRENT_TIMESTAMP"); // Instead of {fn now()} -- see #27534 + return new SQLFragment().appendValue(new SQLFragment.NowTimestamp()); } else if ("_ts".equalsIgnoreCase(getName()) && !getSqlDialect().isSqlServer() && JdbcType.BIGINT == getJdbcType()) { diff --git a/api/src/org/labkey/api/data/SQLFragment.java b/api/src/org/labkey/api/data/SQLFragment.java index 7704f82b272..9e8fffe00ac 100644 --- a/api/src/org/labkey/api/data/SQLFragment.java +++ b/api/src/org/labkey/api/data/SQLFragment.java @@ -1,1343 +1,1365 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.data; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.query.AliasManager; -import org.labkey.api.query.FieldKey; -import org.labkey.api.settings.AppProps; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JdbcUtil; -import org.labkey.api.util.Pair; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.TreeSet; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; - -import static org.labkey.api.query.ExprColumn.STR_TABLE_ALIAS; - -/** - * Holds both the SQL text and JDBC parameter values to use during invocation. - * User: Matthew - * Date: Apr 19, 2006 - */ -public class SQLFragment implements Appendable, CharSequence -{ - public static final String FEATUREFLAG_DISABLE_STRICT_CHECKS = "sqlfragment-disable-strict-checks"; - - private String sql; - private StringBuilder sb = null; - private List params; // TODO: Should be List - - private final List tempTokens = new ArrayList<>(); // Hold refs to ensure they're not GC'd - - // use ordered map to make sql generation more deterministic (see collectCommonTableExpressions()) - private LinkedHashMap commonTableExpressionsMap = null; - - private static class CTE - { - CTE(@NotNull SqlDialect dialect, @NotNull String name) - { - this.dialect = dialect; - this.preferredName = name; - tokens.add("/*$*/" + GUID.makeGUID() + ":" + name + "/*$*/"); - } - - CTE(@NotNull SqlDialect dialect, @NotNull String name, SQLFragment sqlf, boolean recursive) - { - this(dialect, name); - this.sqlf = sqlf; - this.recursive = recursive; - } - - CTE(CTE from) - { - this.dialect = from.dialect; - this.preferredName = from.preferredName; - this.tokens.addAll(from.tokens); - this.sqlf = from.sqlf; - this.recursive = from.recursive; - } - - public CTE copy(boolean deep) - { - CTE copy = new CTE(this); - if (deep) - copy.sqlf = new SQLFragment().append(copy.sqlf); - return copy; - } - - private String token() - { - return tokens.iterator().next(); - } - - private final @NotNull SqlDialect dialect; - final String preferredName; - boolean recursive = false; // NOTE this is dialect dependant (getSql() does not take a dialect) - final Set tokens = new TreeSet<>(); - SQLFragment sqlf = null; - } - - public SQLFragment() - { - sql = ""; - } - - public SQLFragment(CharSequence charseq, @Nullable List params) - { - if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || - (StringUtils.countMatches(charseq, '\"') % 2) != 0 || - StringUtils.contains(charseq, ';')) - { - if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); - } - - // allow statement separators - this.sql = charseq.toString(); - if (null != params) - this.params = new ArrayList<>(params); - } - - - public SQLFragment(CharSequence sql, Object... params) - { - this(sql, Arrays.asList(params)); - } - - - public SQLFragment(SQLFragment other) - { - this(other,false); - } - - - public SQLFragment(SQLFragment other, boolean deep) - { - sql = other.getSqlCharSequence().toString(); - if (null != other.params) - addAll(other.params); - if (null != other.commonTableExpressionsMap && !other.commonTableExpressionsMap.isEmpty()) - { - if (null == this.commonTableExpressionsMap) - this.commonTableExpressionsMap = new LinkedHashMap<>(); - for (Map.Entry e : other.commonTableExpressionsMap.entrySet()) - { - CTE cte = e.getValue().copy(deep); - this.commonTableExpressionsMap.put(e.getKey(),cte); - } - } - this.tempTokens.addAll(other.tempTokens); - } - - - @Override - public boolean isEmpty() - { - return (null == sb || sb.isEmpty()) && (sql == null || sql.isEmpty()); - } - - - /* same as getSQL() but without CTE handling */ - public String getRawSQL() - { - return null != sb ? sb.toString() : null != sql ? sql : ""; - } - - /* - * Directly set the current SQL. - * - * This is useful for wrapping existing SQL, for instance adding a cast - * Obviously parameter number and order must remain unchanged - * - * This can also be used for processing sql scripts (e.g. module .sql update scripts) - */ - public SQLFragment setSqlUnsafe(String unsafe) - { - this.sql = unsafe; - this.sb = null; - return this; - } - - public static SQLFragment unsafe(String unsafe) - { - return new SQLFragment().setSqlUnsafe(unsafe); - } - - - private String replaceCteTokens(String self, String select, List> ctes) - { - for (Pair pair : ctes) - { - String alias = pair.first; - CTE cte = pair.second; - for (String token : cte.tokens) - { - select = Strings.CS.replace(select, token, alias); - } - } - if (null != self) - select = Strings.CS.replace(select, "$SELF$", self); - return select; - } - - - private List collectCommonTableExpressions() - { - List list = new ArrayList<>(); - _collectCommonTableExpressions(list); - return list; - } - - private void _collectCommonTableExpressions(List list) - { - if (null != commonTableExpressionsMap) - { - commonTableExpressionsMap.values().forEach(cte -> cte.sqlf._collectCommonTableExpressions(list)); - list.addAll(commonTableExpressionsMap.values()); - } - } - - - public String getSQL() - { - if (null == commonTableExpressionsMap || commonTableExpressionsMap.isEmpty()) - return null != sb ? sb.toString() : null != sql ? sql : ""; - - List commonTableExpressions = collectCommonTableExpressions(); - assert !commonTableExpressions.isEmpty(); - - boolean recursive = commonTableExpressions.stream() - .anyMatch(cte -> cte.recursive); - StringBuilder ret = new StringBuilder("WITH" + (recursive ? " RECURSIVE" : "")); - - // generate final aliases for each CTE */ - SqlDialect dialect = Objects.requireNonNull(commonTableExpressions.get(0).dialect); - AliasManager am = new AliasManager(dialect); - List> ctes = commonTableExpressions.stream() - .map(cte -> new Pair<>(am.decideAlias(cte.preferredName),cte)) - .collect(Collectors.toList()); - - String comma = "\n/*CTE*/\n\t"; - for (Pair p : ctes) - { - String alias = p.first; - CTE cte = p.second; - SQLFragment expr = cte.sqlf; - String sql = expr._getOwnSql(alias, ctes); - ret.append(comma).append(alias).append(" AS (").append(sql).append(")"); - comma = "\n,/*CTE*/\n\t"; - } - ret.append("\n"); - - String select = _getOwnSql( null, ctes ); - ret.append(replaceCteTokens(null, select, ctes)); - return ret.toString(); - } - - - private String _getOwnSql(String alias, List> ctes) - { - String ownSql = null != sb ? sb.toString() : null != this.sql ? this.sql : ""; - return replaceCteTokens(alias, ownSql, ctes); - } - - - static Pattern markerPattern = Pattern.compile("/\\*\\$\\*/.*/\\*\\$\\*/"); - - /* This is not an exhaustive .equals() test, but it give pretty good confidence that these statements are the same */ - static boolean debugCompareSQL(SQLFragment sql1, SQLFragment sql2) - { - String select1 = sql1.getRawSQL(); - String select2 = sql2.getRawSQL(); - - if ((null == sql1.commonTableExpressionsMap || sql1.commonTableExpressionsMap.isEmpty()) && - (null == sql2.commonTableExpressionsMap || sql2.commonTableExpressionsMap.isEmpty())) - return select1.equals(select2); - - select1 = markerPattern.matcher(select1).replaceAll("CTE"); - select2 = markerPattern.matcher(select2).replaceAll("CTE"); - if (!select1.equals(select2)) - return false; - - Set ctes1 = sql1.commonTableExpressionsMap.values().stream() - .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) - .collect(Collectors.toSet()); - Set ctes2 = sql2.commonTableExpressionsMap.values().stream() - .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) - .collect(Collectors.toSet()); - return ctes1.equals(ctes2); - } - - - // It is a little confusing that getString() does not return the same charsequence that this object purports to - // represent. However, this is a good "display value" for this object. - // see getSqlCharSequence() - @NotNull - public String toString() - { - return "SQLFragment@" + System.identityHashCode(this) + "\n" + JdbcUtil.format(this); - } - - - public String toDebugString() - { - return JdbcUtil.format(this); - } - - - public List getParams() - { - var ctes = collectCommonTableExpressions(); - List ret = new ArrayList<>(); - - for (var cte : ctes) - ret.addAll(cte.sqlf.getParamsNoCTEs()); - ret.addAll(getParamsNoCTEs()); - return Collections.unmodifiableList(ret); - } - - - public List> getParamsWithFragments() - { - var ctes = collectCommonTableExpressions(); - List> ret = new ArrayList<>(); - - for (CTE cte : ctes) - { - if (null != cte.sqlf && null != cte.sqlf.params) - { - for (int i = 0; i < cte.sqlf.params.size(); i++) - { - ret.add(new Pair<>(cte.sqlf, i)); - } - } - } - - if (null != params) - { - for (int i = 0; i < params.size(); i++) - { - ret.add(new Pair<>(this, i)); - } - } - return ret; - } - - private final static Object[] EMPTY_ARRAY = new Object[0]; - - public Object[] getParamsArray() - { - return null == params ? EMPTY_ARRAY : params.toArray(); - } - - public List getParamsNoCTEs() - { - return params == null ? Collections.emptyList() : Collections.unmodifiableList(params); - } - - private List getMutableParams() - { - if (!(params instanceof ArrayList)) - { - List t = new ArrayList<>(); - if (params != null) - t.addAll(params); - params = t; - } - return params; - } - - - private StringBuilder getStringBuilder() - { - if (null == sb) - sb = new StringBuilder(null==sql?"":sql); - return sb; - } - - - @Override - public SQLFragment append(CharSequence charseq) - { - if (null == charseq) - return this; - - if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || - (StringUtils.countMatches(charseq, '\"') % 2) != 0 || - StringUtils.contains(charseq, ';')) - { - if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); - } - - getStringBuilder().append(charseq); - return this; - } - - public SQLFragment appendIdentifier(DatabaseIdentifier id) - { - return append(id.getSql()); - } - - /** Functionally the same as append(CharSequence). This method just has different asserts */ - public SQLFragment appendIdentifier(CharSequence charseq) - { - if (null == charseq) - return this; - if (charseq instanceof SQLFragment sqlf) - { - if (0 != sqlf.getParamsArray().length) - throw new IllegalStateException("Unexpected SQL in appendIdentifier()"); - charseq = sqlf.getRawSQL(); - } - - String identifier = charseq.toString().strip(); - - if (STR_TABLE_ALIAS.equals(identifier)) - { - getStringBuilder().append(identifier); - return this; - } - - boolean malformed; - if (identifier.length() >= 2 && identifier.startsWith("\"") && identifier.endsWith("\"")) - malformed = (StringUtils.countMatches(identifier, '\"') % 2) != 0; - else if (identifier.length() >= 2 && identifier.startsWith("`") && identifier.endsWith("`")) - malformed = (StringUtils.countMatches(identifier, '`') % 2) != 0; - else - malformed = StringUtils.containsAny(identifier, "*/\\'\"`?;- \t\n"); - if (malformed && !AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - throw new IllegalArgumentException("SQLFragment.appendIdentifier(String) value appears to be incorrectly formatted: " + identifier); - - getStringBuilder().append(charseq); - return this; - } - - // just to save some typing - public SQLFragment appendDottedIdentifiers(CharSequence table, DatabaseIdentifier col) - { - return appendIdentifier(table).append(".").appendIdentifier(col); - } - - // just to save some typing - public SQLFragment appendDottedIdentifiers(CharSequence... ids) - { - var dot = ""; - for (var id : ids) - { - append(dot).appendIdentifier(id); - dot = "."; - } - return this; - } - - /** append End Of Statement */ - public SQLFragment appendEOS() - { - getStringBuilder().append(";\n"); - return this; - } - - - @Override - public SQLFragment append(CharSequence csq, int start, int end) - { - append(csq.subSequence(start, end)); - return this; - } - - /** Adds the container's ID as an in-line string constant to the SQL */ - public SQLFragment appendValue(Container c) - { - if (null == c) - return appendNull(); - return appendValue(c, null); - } - - public SQLFragment appendValue(@NotNull Container c, SqlDialect dialect) - { - appendValue(c.getEntityId(), dialect); - String name = c.getName(); - if (!StringUtils.containsAny(name,"*/\\'\"?")) - append("/* ").append(name).append(" */"); - return this; - } - - public SQLFragment appendNull() - { - getStringBuilder().append("NULL"); - return this; - } - - public SQLFragment appendValue(Boolean B, @NotNull SqlDialect dialect) - { - if (null == B) - return append("CAST(NULL AS ").append(dialect.getBooleanDataType()).append(")"); - getStringBuilder().append(B ? dialect.getBooleanTRUE() : dialect.getBooleanFALSE()); - return this; - } - - public SQLFragment appendValue(Integer I) - { - if (null == I) - return appendNull(); - getStringBuilder().append(I.intValue()); - return this; - } - - public SQLFragment appendValue(int i) - { - getStringBuilder().append(i); - return this; - } - - - public SQLFragment appendValue(Long L) - { - if (null == L) - return appendNull(); - getStringBuilder().append((long)L); - return this; - } - - public SQLFragment appendValue(long l) - { - getStringBuilder().append(l); - return this; - } - - public SQLFragment appendValue(Float F) - { - if (null == F) - return appendNull(); - return appendValue(F.floatValue()); - } - - public SQLFragment appendValue(float f) - { - if (Float.isFinite(f)) - { - getStringBuilder().append(f); - } - else - { - getStringBuilder().append("?"); - add(f); - } - return this; - } - - public SQLFragment appendValue(Double D) - { - if (null == D) - return appendNull(); - else - return appendValue(D.doubleValue()); - } - - public SQLFragment appendValue(double d) - { - if (Double.isFinite(d)) - { - getStringBuilder().append(d); - } - else - { - getStringBuilder().append("?"); - add(d); - } - return this; - } - - public SQLFragment appendValue(Number N) - { - if (null == N) - return appendNull(); - - if (N instanceof Quantity q) - N = q.value(); - - if (N instanceof BigDecimal || N instanceof BigInteger || N instanceof Long) - { - getStringBuilder().append(N); - } - else if (Double.isFinite(N.doubleValue())) - { - getStringBuilder().append(N); - } - else - { - getStringBuilder().append(" ? "); - add(N); - } - return this; - } - - public final SQLFragment appendValue(java.util.Date d) - { - if (null == d) - return appendNull(); - if (d.getClass() == java.util.Date.class) - getStringBuilder().append("{ts '").append(new Timestamp(d.getTime())).append("'}"); - else if (d.getClass() == java.sql.Timestamp.class) - getStringBuilder().append("{ts '").append(d).append("'}"); - else if (d.getClass() == java.sql.Date.class) - getStringBuilder().append("{d '").append(d).append("'}"); - else - throw new IllegalStateException("Unexpected date type: " + d.getClass().getName()); - return this; - } - - public SQLFragment appendValue(GUID g) - { - return appendValue(g, null); - } - - public SQLFragment appendValue(GUID g, SqlDialect d) - { - if (null == g) - return appendNull(); - // doesn't need StringHandler, just hex and hyphen - String sqlGUID = "'" + g + "'"; - // I'm testing dialect type, because some dialects do not support getGuidType(), and postgers uses VARCHAR anyway - if (null != d && d.isSqlServer()) - getStringBuilder().append("CAST(").append(sqlGUID).append(" AS UNIQUEIDENTIFIER)"); - else - getStringBuilder().append(sqlGUID); - return this; - } - - public SQLFragment appendValue(Enum e) - { - if (null == e) - return appendNull(); - String name = e.name(); - // Enum.name() usually returns a simple string (a legal java identifier), this is a paranoia check. - if (name.contains("'")) - throw new IllegalStateException(); - getStringBuilder().append("'").append(name).append("'"); - return this; - } - - public SQLFragment append(FieldKey fk) - { - if (null == fk) - return appendNull(); - append(String.valueOf(fk)); - return this; - } - - - /** Adds the object as a JDBC parameter value */ - public SQLFragment add(Object p) - { - getMutableParams().add(p); - return this; - } - - - /** Adds the objects as JDBC parameter values */ - public SQLFragment addAll(Collection l) - { - getMutableParams().addAll(l); - return this; - } - - - /** Adds the objects as JDBC parameter values */ - public SQLFragment addAll(Object... values) - { - if (values == null) - return this; - addAll(Arrays.asList(values)); - return this; - } - - - /** Sets the parameter at the index to the object's value */ - public void set(int i, Object p) - { - getMutableParams().set(i,p); - } - - /** Append both the SQL and the parameters from the other SQLFragment to this SQLFragment */ - public SQLFragment append(SQLFragment f) - { - if (null != f.sb) - getStringBuilder().append(f.sb); - else - getStringBuilder().append(f.sql); - if (null != f.params) - addAll(f.params); - mergeCommonTableExpressions(f); - tempTokens.addAll(f.tempTokens); - return this; - } - - public SQLFragment append(@NotNull Iterable fragments, @NotNull String separator) - { - String s = ""; - for (SQLFragment fragment : fragments) - { - append(s); - s = separator; - append(fragment); - } - return this; - } - - // return boolean so this can be used in an assert. passing in a dialect is not ideal, but parsing comments out - // before submitting the fragment is not reliable and holding statements & comments separately (to eliminate the - // need to parse them) isn't particularly easy... so punt for now. - public boolean appendComment(String comment, SqlDialect dialect) - { - if (dialect.supportsComments()) - { - StringBuilder sb = getStringBuilder(); - int len = sb.length(); - if (len > 0 && sb.charAt(len-1) != '\n') - sb.append('\n'); - sb.append("\n-- "); - boolean truncated = comment.length() > 1000; - if (truncated) - comment = comment.substring(0,1000); - sb.append(comment); - if (StringUtils.countMatches(comment, "'")%2==1) - sb.append("'"); - if (truncated) - sb.append("..."); - sb.append('\n'); - } - return true; - } - - - /** see also append(TableInfo, String alias) */ - public SQLFragment append(TableInfo table) - { - SQLFragment s = table.getSQLName(); - if (s != null) - return append(s); - - String alias = table.getSqlDialect().makeLegalIdentifier(table.getName()); - return append(table.getFromSQL(alias)); - } - - /** Add a table/query to the SQL with an alias, as used in a FROM clause */ - public SQLFragment append(TableInfo table, String alias) - { - return append(table.getFromSQL(alias)); - } - - /** Add to the SQL */ - @Override - public SQLFragment append(char ch) - { - getStringBuilder().append(ch); - return this; - } - - /** This is like appendValue(CharSequence s), but force use of literal syntax - * CAUTIONARY NOTE: String literals in PostgresSQL are tricky because of overloaded functions - * array_agg('string') fails array_agg('string'::VARCHAR) works - * json_object('{}) works json_object('string'::VARCHAR) fails - * In the case of json_object() it expects TEXT. Postgres will promote 'json' to TEXT, but not 'json'::VARCHAR - */ - public SQLFragment appendStringLiteral(CharSequence s, @NotNull SqlDialect d) - { - if (null==s) - return appendNull(); - getStringBuilder().append(d.getStringHandler().quoteStringLiteral(s.toString())); - return this; - } - - /** Add to the SQL as either an in-line string literal or as a JDBC parameter depending on whether it would need escaping */ - public SQLFragment appendValue(CharSequence s) - { - return appendValue(s, null); - } - - public SQLFragment appendValue(CharSequence s, SqlDialect d) - { - if (null==s) - return appendNull(); - if (null==d || s.length() > 200) - return append("?").add(s.toString()); - appendStringLiteral(s, d); - return this; - } - - public SQLFragment appendInClause(@NotNull Collection params, SqlDialect dialect) - { - dialect.appendInClauseSql(this, params); - return this; - } - - public CharSequence getSqlCharSequence() - { - if (null != sb) - { - return sb; - } - return sql; - } - - public void insert(int index, SQLFragment sql) - { - if (!sql.getParams().isEmpty()) - { - throw new IllegalArgumentException("Not supported for SQLFragments with parameters - they must be inserted/merged separately"); - } - if (sql.commonTableExpressionsMap != null && !sql.commonTableExpressionsMap.isEmpty()) - { - throw new IllegalArgumentException("Not supported for SQLFragments with CTEs - they must be inserted/merged separately"); - } - if (!tempTokens.isEmpty()) - { - throw new IllegalArgumentException("Not supported for SQLFragments with temp tokens - they must be inserted/merged separately"); - } - getStringBuilder().insert(index, sql.getRawSQL()); - } - - /** Insert into the SQL */ - public void insert(int index, String str) - { - if ((StringUtils.countMatches(str, '\'') % 2) != 0 || - (StringUtils.countMatches(str, '\"') % 2) != 0 || - StringUtils.contains(str, ';')) - { - if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - throw new IllegalArgumentException("SQLFragment.insert(int,String) does not allow semicolons or unmatched quotes"); - } - - getStringBuilder().insert(index, str); - } - - /** Insert this SQLFragment's SQL and parameters at the start of the existing SQL and parameters */ - public void prepend(SQLFragment sql) - { - getStringBuilder().insert(0, sql.getSqlCharSequence().toString()); - if (null != sql.params) - getMutableParams().addAll(0, sql.params); - mergeCommonTableExpressions(sql); - } - - - public int indexOf(String str) - { - return getStringBuilder().indexOf(str); - } - - - // Display query in "English" (display SQL with params substituted) - // with a little more work could probably be made to be SQL legal - public String getFilterText() - { - String sql = getSQL().replaceFirst("WHERE ", ""); - List params = getParams(); - for (Object param1 : params) - { - String param = param1.toString(); - param = param.replaceAll("\\\\", "\\\\\\\\"); - param = param.replaceAll("\\$", "\\\\\\$"); - sql = sql.replaceFirst("\\?", param); - } - return sql.replaceAll("\"", ""); - } - - - @Override - public char charAt(int index) - { - return getSqlCharSequence().charAt(index); - } - - @Override - public int length() - { - return getSqlCharSequence().length(); - } - - @Override - public @NotNull CharSequence subSequence(int start, int end) - { - return getSqlCharSequence().subSequence(start, end); - } - - /** - * KEY is used as a faster way to look for equivalent CTE expressions. - * returning a name here allows us to potentially merge CTE at add time - * - * if you don't have a key you can just use sqlf.toString() - */ - public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf) - { - return addCommonTableExpression(dialect, key, proposedName, sqlf, false); - } - - public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf, boolean recursive) - { - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - CTE prev = commonTableExpressionsMap.get(key); - if (null != prev) - return prev.token(); - CTE cte = new CTE(dialect, proposedName, sqlf, recursive); - commonTableExpressionsMap.put(key, cte); - return cte.token(); - } - - public String createCommonTableExpressionToken(SqlDialect dialect, Object key, String proposedName) - { - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - CTE prev = commonTableExpressionsMap.get(key); - if (null != prev) - throw new IllegalStateException("Cannot create CTE token from already used key."); - CTE cte = new CTE(dialect ,proposedName); - commonTableExpressionsMap.put(key, cte); - return cte.token(); - } - - public void setCommonTableExpressionSql(Object key, SQLFragment sqlf, boolean recursive) - { - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - - if (null != sqlf.commonTableExpressionsMap && !sqlf.commonTableExpressionsMap.isEmpty()) - { - // Need to merge CTEs up; this.cte depends on newSql.ctes, so they need to come first - SQLFragment newSql = new SQLFragment(sqlf); - LinkedHashMap toMap = new LinkedHashMap<>(newSql.commonTableExpressionsMap); - for (Map.Entry e : commonTableExpressionsMap.entrySet()) - { - CTE from = e.getValue(); - CTE to = toMap.get(e.getKey()); - if (null != to) - to.tokens.addAll(from.tokens); - else - toMap.put(e.getKey(), from.copy(false)); - } - - commonTableExpressionsMap = toMap; - newSql.commonTableExpressionsMap = null; - sqlf = newSql; - } - - CTE cte = commonTableExpressionsMap.get(key); - if (null == cte) - throw new IllegalStateException("CTE not found."); - cte.sqlf = sqlf; - cte.recursive = recursive; - } - - - private void mergeCommonTableExpressions(SQLFragment sqlFrom) - { - if (null == sqlFrom.commonTableExpressionsMap || sqlFrom.commonTableExpressionsMap.isEmpty()) - return; - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - for (Map.Entry e : sqlFrom.commonTableExpressionsMap.entrySet()) - { - CTE from = e.getValue(); - CTE to = commonTableExpressionsMap.get(e.getKey()); - if (null != to) - to.tokens.addAll(from.tokens); - else - commonTableExpressionsMap.put(e.getKey(), from.copy(false)); - } - } - - - public void addTempToken(Object tempToken) - { - tempTokens.add(tempToken); - } - - public void addTempTokens(SQLFragment other) - { - tempTokens.add(other.tempTokens); - } - - public static SQLFragment prettyPrint(SQLFragment from) - { - SQLFragment sqlf = new SQLFragment(from); - - String s = from.getSqlCharSequence().toString(); - StringBuilder sb = new StringBuilder(s.length() + 200); - String[] lines = StringUtils.split(s, '\n'); - int indent = 0; - - for (String line : lines) - { - String t = line.trim(); - - if (t.isEmpty()) - continue; - - if (t.startsWith("-- params = b.getParams(); - assertEquals(2,params.size()); - assertEquals(5, params.get(0)); - assertEquals("xxyzzy", params.get(1)); - - - SQLFragment c = new SQLFragment(b); - assertEquals(""" - WITH - /*CTE*/ - \tCTE AS (SELECT a FROM b WHERE x=?) - SELECT * FROM CTE WHERE y=?""", - c.getSQL()); - assertEquals(""" - WITH - /*CTE*/ - \tCTE AS (SELECT a FROM b WHERE x=5) - SELECT * FROM CTE WHERE y='xxyzzy'""", - filterDebugString(c.toDebugString())); - params = c.getParams(); - assertEquals(2,params.size()); - assertEquals(5, params.get(0)); - assertEquals("xxyzzy", params.get(1)); - - - // combining - - SQLFragment sqlf = new SQLFragment(); - String token = sqlf.addCommonTableExpression(dialect, "KEY_A", "cte1", new SQLFragment("SELECT * FROM a")); - sqlf.append("SELECT * FROM ").append(token).append(" _1"); - - assertEquals(""" - WITH - /*CTE*/ - \tcte1 AS (SELECT * FROM a) - SELECT * FROM cte1 _1""", - sqlf.getSQL()); - - SQLFragment sqlf2 = new SQLFragment(); - String token2 = sqlf2.addCommonTableExpression(dialect, "KEY_A", "cte2", new SQLFragment("SELECT * FROM a")); - sqlf2.append("SELECT * FROM ").append(token2).append(" _2"); - assertEquals(""" - WITH - /*CTE*/ - \tcte2 AS (SELECT * FROM a) - SELECT * FROM cte2 _2""", - sqlf2.getSQL()); - - SQLFragment sqlf3 = new SQLFragment(); - String token3 = sqlf3.addCommonTableExpression(dialect, "KEY_B", "cte3", new SQLFragment("SELECT * FROM b")); - sqlf3.append("SELECT * FROM ").append(token3).append(" _3"); - assertEquals(""" - WITH - /*CTE*/ - \tcte3 AS (SELECT * FROM b) - SELECT * FROM cte3 _3""", - sqlf3.getSQL()); - - SQLFragment union = new SQLFragment(); - union.append(sqlf); - union.append("\nUNION\n"); - union.append(sqlf2); - union.append("\nUNION\n"); - union.append(sqlf3); - assertEquals(""" - WITH - /*CTE*/ - \tcte1 AS (SELECT * FROM a) - ,/*CTE*/ - \tcte3 AS (SELECT * FROM b) - SELECT * FROM cte1 _1 - UNION - SELECT * FROM cte1 _2 - UNION - SELECT * FROM cte3 _3""", - union.getSQL()); - } - - @Test - public void nested_cte() - { - // one-level cte using cteToken (CTE fragment 'a' does not contain a CTE) - { - SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); - assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); - SQLFragment b = new SQLFragment(); - String cteToken = b.addCommonTableExpression(dialect, new Object(), "CTE", a); - b.append("SELECT * FROM ").append(cteToken).append(" WHERE p=?").add("parameterTWO"); - assertEquals(""" - WITH - /*CTE*/ - \tCTE AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) - SELECT * FROM CTE WHERE p='parameterTWO'""", - filterDebugString(b.toDebugString())); - assertEquals("parameterONE", b.getParams().get(0)); - } - - // two-level cte using cteTokens (CTE fragment 'b' contains a CTE of fragment a) - { - SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); - assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); - SQLFragment b = new SQLFragment(); - String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); - b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterTWO"); - SQLFragment c = new SQLFragment(); - String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); - c.append("SELECT * FROM ").append(cteTokenB).append(" WHERE i=?").add(3); - assertEquals(""" - WITH - /*CTE*/ - \tA_ AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) - ,/*CTE*/ - \tB_ AS (SELECT * FROM A_ WHERE p='parameterTWO') - SELECT * FROM B_ WHERE i=3""", - filterDebugString(c.toDebugString())); - List params = c.getParams(); - assertEquals(3, params.size()); - assertEquals("parameterONE", params.get(0)); - assertEquals("parameterTWO", params.get(1)); - assertEquals(3, params.get(2)); - } - - // Same as previous but top-level query has both a nested and non-nested CTE - { - SQLFragment a = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); - SQLFragment a2 = new SQLFragment("SELECT 2 as i, 'Atwo' as s, CAST(? AS VARCHAR) as p", "parameterAtwo"); - SQLFragment b = new SQLFragment(); - String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); - b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); - SQLFragment c = new SQLFragment(); - String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); - String cteTokenA2 = c.addCommonTableExpression(dialect, new Object(), "A2_", a2); - c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); - assertEquals(""" - WITH - /*CTE*/ - \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) - ,/*CTE*/ - \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') - ,/*CTE*/ - \tA2_ AS (SELECT 2 as i, 'Atwo' as s, CAST('parameterAtwo' AS VARCHAR) as p) - SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", - filterDebugString(c.toDebugString())); - List params = c.getParams(); - assertEquals(4, params.size()); - assertEquals("parameterAone", params.get(0)); - assertEquals("parameterB", params.get(1)); - assertEquals("parameterAtwo", params.get(2)); - assertEquals(4, params.get(3)); - } - - // Same as previous but two of the CTEs are the same and should be collapsed (e.g. imagine a container filter implemented with a CTE) - // TODO, we only collapse CTEs that are siblings - { - SQLFragment cf = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); - SQLFragment b = new SQLFragment(); - String cteTokenA = b.addCommonTableExpression(dialect, "CTE_KEY_CF", "A_", cf); - b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); - SQLFragment c = new SQLFragment(); - String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); - String cteTokenA2 = c.addCommonTableExpression(dialect, "CTE_KEY_CF", "A2_", cf); - c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); - assertEquals(""" - WITH - /*CTE*/ - \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) - ,/*CTE*/ - \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') - ,/*CTE*/ - \tA2_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) - SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", - filterDebugString(c.toDebugString())); - List params = c.getParams(); - assertEquals(4, params.size()); - assertEquals("parameterAone", params.get(0)); - assertEquals("parameterB", params.get(1)); - assertEquals("parameterAone", params.get(2)); - assertEquals(4, params.get(3)); - } - } - - - private void shouldFail(Runnable r) - { - try - { - r.run(); - if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - fail("Expected IllegalArgumentException"); - } - catch (IllegalArgumentException e) - { - if (AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - fail("Did not expect IllegalArgumentException"); - } - } - - - @Test - public void testIllegalArgument() - { - shouldFail(() -> new SQLFragment(";")); - shouldFail(() -> new SQLFragment().append(";")); - shouldFail(() -> new SQLFragment("AND name='")); - shouldFail(() -> new SQLFragment().append("AND name = '")); - shouldFail(() -> new SQLFragment().append("AND name = 'Robert'); DROP TABLE Students; --")); - - shouldFail(() -> new SQLFragment().appendIdentifier("column name")); - shouldFail(() -> new SQLFragment().appendIdentifier("?")); - shouldFail(() -> new SQLFragment().appendIdentifier(";")); - shouldFail(() -> new SQLFragment().appendIdentifier("\"column\"name\"")); - } - - - String mysqlQuoteIdentifier(String id) - { - return "`" + id.replaceAll("`", "``") + "`"; - } - - @Test - public void testMysql() - { - // OK - new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("mysql")); - new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my`sql")); - new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my\"sql")); - - // not OK - shouldFail(() -> new SQLFragment().appendIdentifier("`")); - shouldFail(() -> new SQLFragment().appendIdentifier("`a`a`")); - } - } - - - public static class IntegrationTestCase extends Assert - { - @Test - public void test() - { - // try some Dialect stuff and CTE executed against core schema - } - } - - - @Override - public boolean equals(Object obj) - { - if (!(obj instanceof SQLFragment other)) - { - return false; - } - return getSQL().equals(other.getSQL()) && getParams().equals(other.getParams()); - } - - /** - * Joins the SQLFragments in the provided {@code Iterable} into a single SQLFragment. The SQL is joined by string - * concatenation using the provided separator. The parameters are combined to form the new parameter list. - * - * @param fragments SQLFragments to join together - * @param separator Separator to use on the SQL portion - * @return A new SQLFragment that joins all the SQLFragments - */ - public static SQLFragment join(Iterable fragments, String separator) - { - if (separator.contains("?")) - throw new IllegalStateException("separator must not include a parameter marker"); - - // Join all the SQL statements - String sql = StreamSupport.stream(fragments.spliterator(), false) - .map(SQLFragment::getSQL) - .collect(Collectors.joining(separator)); - - // Collect all the parameters to a single list - List params = StreamSupport.stream(fragments.spliterator(), false) - .map(SQLFragment::getParams) - .flatMap(Collection::stream) - .collect(Collectors.toList()); - - return new SQLFragment(sql, params); - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.data; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.query.AliasManager; +import org.labkey.api.query.FieldKey; +import org.labkey.api.settings.AppProps; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JdbcUtil; +import org.labkey.api.util.Pair; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +import static org.labkey.api.query.ExprColumn.STR_TABLE_ALIAS; + +/** + * Holds both the SQL text and JDBC parameter values to use during invocation. + */ +public class SQLFragment implements Appendable, CharSequence +{ + public static final String FEATUREFLAG_DISABLE_STRICT_CHECKS = "sqlfragment-disable-strict-checks"; + + private String sql; + private StringBuilder sb = null; + private List params; // TODO: Should be List + + private final List tempTokens = new ArrayList<>(); // Hold refs to ensure they're not GC'd + + // use ordered map to make sql generation more deterministic (see collectCommonTableExpressions()) + private LinkedHashMap commonTableExpressionsMap = null; + + private static class CTE + { + CTE(@NotNull SqlDialect dialect, @NotNull String name) + { + this.dialect = dialect; + this.preferredName = name; + tokens.add("/*$*/" + GUID.makeGUID() + ":" + name + "/*$*/"); + } + + CTE(@NotNull SqlDialect dialect, @NotNull String name, SQLFragment sqlf, boolean recursive) + { + this(dialect, name); + this.sqlf = sqlf; + this.recursive = recursive; + } + + CTE(CTE from) + { + this.dialect = from.dialect; + this.preferredName = from.preferredName; + this.tokens.addAll(from.tokens); + this.sqlf = from.sqlf; + this.recursive = from.recursive; + } + + public CTE copy(boolean deep) + { + CTE copy = new CTE(this); + if (deep) + copy.sqlf = new SQLFragment().append(copy.sqlf); + return copy; + } + + private String token() + { + return tokens.iterator().next(); + } + + private final @NotNull SqlDialect dialect; + final String preferredName; + boolean recursive = false; // NOTE this is dialect dependant (getSql() does not take a dialect) + final Set tokens = new TreeSet<>(); + SQLFragment sqlf = null; + } + + public SQLFragment() + { + sql = ""; + } + + public SQLFragment(CharSequence charseq, @Nullable List params) + { + if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || + (StringUtils.countMatches(charseq, '\"') % 2) != 0 || + StringUtils.contains(charseq, ';')) + { + if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); + } + + // allow statement separators + this.sql = charseq.toString(); + if (null != params) + this.params = new ArrayList<>(params); + } + + + public SQLFragment(CharSequence sql, Object... params) + { + this(sql, Arrays.asList(params)); + } + + + public SQLFragment(SQLFragment other) + { + this(other,false); + } + + + public SQLFragment(SQLFragment other, boolean deep) + { + sql = other.getSqlCharSequence().toString(); + if (null != other.params) + addAll(other.params); + if (null != other.commonTableExpressionsMap && !other.commonTableExpressionsMap.isEmpty()) + { + if (null == this.commonTableExpressionsMap) + this.commonTableExpressionsMap = new LinkedHashMap<>(); + for (Map.Entry e : other.commonTableExpressionsMap.entrySet()) + { + CTE cte = e.getValue().copy(deep); + this.commonTableExpressionsMap.put(e.getKey(),cte); + } + } + this.tempTokens.addAll(other.tempTokens); + } + + + @Override + public boolean isEmpty() + { + return (null == sb || sb.isEmpty()) && (sql == null || sql.isEmpty()); + } + + + /* same as getSQL() but without CTE handling */ + public String getRawSQL() + { + return null != sb ? sb.toString() : null != sql ? sql : ""; + } + + /* + * Directly set the current SQL. + * + * This is useful for wrapping existing SQL, for instance adding a cast + * Obviously parameter number and order must remain unchanged + * + * This can also be used for processing sql scripts (e.g. module .sql update scripts) + */ + public SQLFragment setSqlUnsafe(String unsafe) + { + this.sql = unsafe; + this.sb = null; + return this; + } + + public static SQLFragment unsafe(String unsafe) + { + return new SQLFragment().setSqlUnsafe(unsafe); + } + + + private String replaceCteTokens(String self, String select, List> ctes) + { + for (Pair pair : ctes) + { + String alias = pair.first; + CTE cte = pair.second; + for (String token : cte.tokens) + { + select = Strings.CS.replace(select, token, alias); + } + } + if (null != self) + select = Strings.CS.replace(select, "$SELF$", self); + return select; + } + + + private List collectCommonTableExpressions() + { + List list = new ArrayList<>(); + _collectCommonTableExpressions(list); + return list; + } + + private void _collectCommonTableExpressions(List list) + { + if (null != commonTableExpressionsMap) + { + commonTableExpressionsMap.values().forEach(cte -> cte.sqlf._collectCommonTableExpressions(list)); + list.addAll(commonTableExpressionsMap.values()); + } + } + + + public String getSQL() + { + if (null == commonTableExpressionsMap || commonTableExpressionsMap.isEmpty()) + return null != sb ? sb.toString() : null != sql ? sql : ""; + + List commonTableExpressions = collectCommonTableExpressions(); + assert !commonTableExpressions.isEmpty(); + + boolean recursive = commonTableExpressions.stream() + .anyMatch(cte -> cte.recursive); + StringBuilder ret = new StringBuilder("WITH" + (recursive ? " RECURSIVE" : "")); + + // generate final aliases for each CTE */ + SqlDialect dialect = Objects.requireNonNull(commonTableExpressions.get(0).dialect); + AliasManager am = new AliasManager(dialect); + List> ctes = commonTableExpressions.stream() + .map(cte -> new Pair<>(am.decideAlias(cte.preferredName),cte)) + .collect(Collectors.toList()); + + String comma = "\n/*CTE*/\n\t"; + for (Pair p : ctes) + { + String alias = p.first; + CTE cte = p.second; + SQLFragment expr = cte.sqlf; + String sql = expr._getOwnSql(alias, ctes); + ret.append(comma).append(alias).append(" AS (").append(sql).append(")"); + comma = "\n,/*CTE*/\n\t"; + } + ret.append("\n"); + + String select = _getOwnSql( null, ctes ); + ret.append(replaceCteTokens(null, select, ctes)); + return ret.toString(); + } + + + private String _getOwnSql(String alias, List> ctes) + { + String ownSql = null != sb ? sb.toString() : null != this.sql ? this.sql : ""; + return replaceCteTokens(alias, ownSql, ctes); + } + + + static Pattern markerPattern = Pattern.compile("/\\*\\$\\*/.*/\\*\\$\\*/"); + + /* This is not an exhaustive .equals() test, but it give pretty good confidence that these statements are the same */ + static boolean debugCompareSQL(SQLFragment sql1, SQLFragment sql2) + { + String select1 = sql1.getRawSQL(); + String select2 = sql2.getRawSQL(); + + if ((null == sql1.commonTableExpressionsMap || sql1.commonTableExpressionsMap.isEmpty()) && + (null == sql2.commonTableExpressionsMap || sql2.commonTableExpressionsMap.isEmpty())) + return select1.equals(select2); + + select1 = markerPattern.matcher(select1).replaceAll("CTE"); + select2 = markerPattern.matcher(select2).replaceAll("CTE"); + if (!select1.equals(select2)) + return false; + + Set ctes1 = sql1.commonTableExpressionsMap.values().stream() + .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) + .collect(Collectors.toSet()); + Set ctes2 = sql2.commonTableExpressionsMap.values().stream() + .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) + .collect(Collectors.toSet()); + return ctes1.equals(ctes2); + } + + + // It is a little confusing that getString() does not return the same charsequence that this object purports to + // represent. However, this is a good "display value" for this object. + // see getSqlCharSequence() + @NotNull + public String toString() + { + return "SQLFragment@" + System.identityHashCode(this) + "\n" + JdbcUtil.format(this); + } + + + public String toDebugString() + { + return JdbcUtil.format(this); + } + + + public List getParams() + { + var ctes = collectCommonTableExpressions(); + List ret = new ArrayList<>(); + + for (var cte : ctes) + ret.addAll(cte.sqlf.getParamsNoCTEs()); + ret.addAll(getParamsNoCTEs()); + return Collections.unmodifiableList(ret); + } + + + public List> getParamsWithFragments() + { + var ctes = collectCommonTableExpressions(); + List> ret = new ArrayList<>(); + + for (CTE cte : ctes) + { + if (null != cte.sqlf && null != cte.sqlf.params) + { + for (int i = 0; i < cte.sqlf.params.size(); i++) + { + ret.add(new Pair<>(cte.sqlf, i)); + } + } + } + + if (null != params) + { + for (int i = 0; i < params.size(); i++) + { + ret.add(new Pair<>(this, i)); + } + } + return ret; + } + + private final static Object[] EMPTY_ARRAY = new Object[0]; + + public Object[] getParamsArray() + { + return null == params ? EMPTY_ARRAY : params.toArray(); + } + + public List getParamsNoCTEs() + { + return params == null ? Collections.emptyList() : Collections.unmodifiableList(params); + } + + private List getMutableParams() + { + if (!(params instanceof ArrayList)) + { + List t = new ArrayList<>(); + if (params != null) + t.addAll(params); + params = t; + } + return params; + } + + + private StringBuilder getStringBuilder() + { + if (null == sb) + sb = new StringBuilder(null==sql?"":sql); + return sb; + } + + + @Override + public SQLFragment append(CharSequence charseq) + { + if (null == charseq) + return this; + + if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || + (StringUtils.countMatches(charseq, '\"') % 2) != 0 || + StringUtils.contains(charseq, ';')) + { + if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); + } + + getStringBuilder().append(charseq); + return this; + } + + public SQLFragment appendIdentifier(DatabaseIdentifier id) + { + return append(id.getSql()); + } + + /** Functionally the same as append(CharSequence). This method just has different asserts */ + public SQLFragment appendIdentifier(CharSequence charseq) + { + if (null == charseq) + return this; + if (charseq instanceof SQLFragment sqlf) + { + if (0 != sqlf.getParamsArray().length) + throw new IllegalStateException("Unexpected SQL in appendIdentifier()"); + charseq = sqlf.getRawSQL(); + } + + String identifier = charseq.toString().strip(); + + if (STR_TABLE_ALIAS.equals(identifier)) + { + getStringBuilder().append(identifier); + return this; + } + + boolean malformed; + if (identifier.length() >= 2 && identifier.startsWith("\"") && identifier.endsWith("\"")) + malformed = (StringUtils.countMatches(identifier, '\"') % 2) != 0; + else if (identifier.length() >= 2 && identifier.startsWith("`") && identifier.endsWith("`")) + malformed = (StringUtils.countMatches(identifier, '`') % 2) != 0; + else + malformed = StringUtils.containsAny(identifier, "*/\\'\"`?;- \t\n"); + if (malformed && !AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + throw new IllegalArgumentException("SQLFragment.appendIdentifier(String) value appears to be incorrectly formatted: " + identifier); + + getStringBuilder().append(charseq); + return this; + } + + // just to save some typing + public SQLFragment appendDottedIdentifiers(CharSequence table, DatabaseIdentifier col) + { + return appendIdentifier(table).append(".").appendIdentifier(col); + } + + // just to save some typing + public SQLFragment appendDottedIdentifiers(CharSequence... ids) + { + var dot = ""; + for (var id : ids) + { + append(dot).appendIdentifier(id); + dot = "."; + } + return this; + } + + /** append End Of Statement */ + public SQLFragment appendEOS() + { + getStringBuilder().append(";\n"); + return this; + } + + + @Override + public SQLFragment append(CharSequence csq, int start, int end) + { + append(csq.subSequence(start, end)); + return this; + } + + /** Adds the container's ID as an in-line string constant to the SQL */ + public SQLFragment appendValue(Container c) + { + if (null == c) + return appendNull(); + return appendValue(c, null); + } + + public SQLFragment appendValue(@NotNull Container c, SqlDialect dialect) + { + appendValue(c.getEntityId(), dialect); + String name = c.getName(); + if (!StringUtils.containsAny(name,"*/\\'\"?")) + append("/* ").append(name).append(" */"); + return this; + } + + public SQLFragment appendNull() + { + getStringBuilder().append("NULL"); + return this; + } + + public SQLFragment appendValue(Boolean B, @NotNull SqlDialect dialect) + { + if (null == B) + return append("CAST(NULL AS ").append(dialect.getBooleanDataType()).append(")"); + getStringBuilder().append(B ? dialect.getBooleanTRUE() : dialect.getBooleanFALSE()); + return this; + } + + public SQLFragment appendValue(Integer I) + { + if (null == I) + return appendNull(); + getStringBuilder().append(I.intValue()); + return this; + } + + public SQLFragment appendValue(int i) + { + getStringBuilder().append(i); + return this; + } + + + public SQLFragment appendValue(Long L) + { + if (null == L) + return appendNull(); + getStringBuilder().append((long)L); + return this; + } + + public SQLFragment appendValue(long l) + { + getStringBuilder().append(l); + return this; + } + + public SQLFragment appendValue(Float F) + { + if (null == F) + return appendNull(); + return appendValue(F.floatValue()); + } + + public SQLFragment appendValue(float f) + { + if (Float.isFinite(f)) + { + getStringBuilder().append(f); + } + else + { + getStringBuilder().append("?"); + add(f); + } + return this; + } + + public SQLFragment appendValue(Double D) + { + if (null == D) + return appendNull(); + else + return appendValue(D.doubleValue()); + } + + public SQLFragment appendValue(double d) + { + if (Double.isFinite(d)) + { + getStringBuilder().append(d); + } + else + { + getStringBuilder().append("?"); + add(d); + } + return this; + } + + public SQLFragment appendValue(Number N) + { + if (null == N) + return appendNull(); + + if (N instanceof Quantity q) + N = q.value(); + + if (N instanceof BigDecimal || N instanceof BigInteger || N instanceof Long) + { + getStringBuilder().append(N); + } + else if (Double.isFinite(N.doubleValue())) + { + getStringBuilder().append(N); + } + else + { + getStringBuilder().append(" ? "); + add(N); + } + return this; + } + + // Issue 27534: Stop using {fn now()} in function declarations + // Issue 48864: Query Table's use of web server time can cause discrepancies in created/modified timestamps + public final SQLFragment appendValue(NowTimestamp now) + { + if (null == now) + return appendNull(); + getStringBuilder().append("CURRENT_TIMESTAMP"); + return this; + } + + public final SQLFragment appendValue(java.util.Date d) + { + if (null == d) + return appendNull(); + if (d.getClass() == java.util.Date.class) + getStringBuilder().append("{ts '").append(new Timestamp(d.getTime())).append("'}"); + else if (d.getClass() == java.sql.Timestamp.class) + getStringBuilder().append("{ts '").append(d).append("'}"); + else if (d.getClass() == java.sql.Date.class) + getStringBuilder().append("{d '").append(d).append("'}"); + else + throw new IllegalStateException("Unexpected date type: " + d.getClass().getName()); + return this; + } + + public SQLFragment appendValue(GUID g) + { + return appendValue(g, null); + } + + public SQLFragment appendValue(GUID g, SqlDialect d) + { + if (null == g) + return appendNull(); + // doesn't need StringHandler, just hex and hyphen + String sqlGUID = "'" + g + "'"; + // I'm testing dialect type, because some dialects do not support getGuidType(), and postgers uses VARCHAR anyway + if (null != d && d.isSqlServer()) + getStringBuilder().append("CAST(").append(sqlGUID).append(" AS UNIQUEIDENTIFIER)"); + else + getStringBuilder().append(sqlGUID); + return this; + } + + public SQLFragment appendValue(Enum e) + { + if (null == e) + return appendNull(); + String name = e.name(); + // Enum.name() usually returns a simple string (a legal java identifier), this is a paranoia check. + if (name.contains("'")) + throw new IllegalStateException(); + getStringBuilder().append("'").append(name).append("'"); + return this; + } + + public SQLFragment append(FieldKey fk) + { + if (null == fk) + return appendNull(); + append(String.valueOf(fk)); + return this; + } + + + /** Adds the object as a JDBC parameter value */ + public SQLFragment add(Object p) + { + getMutableParams().add(p); + return this; + } + + + /** Adds the objects as JDBC parameter values */ + public SQLFragment addAll(Collection l) + { + getMutableParams().addAll(l); + return this; + } + + + /** Adds the objects as JDBC parameter values */ + public SQLFragment addAll(Object... values) + { + if (values == null) + return this; + addAll(Arrays.asList(values)); + return this; + } + + + /** Sets the parameter at the index to the object's value */ + public void set(int i, Object p) + { + getMutableParams().set(i,p); + } + + /** Append both the SQL and the parameters from the other SQLFragment to this SQLFragment */ + public SQLFragment append(SQLFragment f) + { + if (null != f.sb) + getStringBuilder().append(f.sb); + else + getStringBuilder().append(f.sql); + if (null != f.params) + addAll(f.params); + mergeCommonTableExpressions(f); + tempTokens.addAll(f.tempTokens); + return this; + } + + public SQLFragment append(@NotNull Iterable fragments, @NotNull String separator) + { + String s = ""; + for (SQLFragment fragment : fragments) + { + append(s); + s = separator; + append(fragment); + } + return this; + } + + // return boolean so this can be used in an assert. passing in a dialect is not ideal, but parsing comments out + // before submitting the fragment is not reliable and holding statements & comments separately (to eliminate the + // need to parse them) isn't particularly easy... so punt for now. + public boolean appendComment(String comment, SqlDialect dialect) + { + if (dialect.supportsComments()) + { + StringBuilder sb = getStringBuilder(); + int len = sb.length(); + if (len > 0 && sb.charAt(len-1) != '\n') + sb.append('\n'); + sb.append("\n-- "); + boolean truncated = comment.length() > 1000; + if (truncated) + comment = comment.substring(0,1000); + sb.append(comment); + if (StringUtils.countMatches(comment, "'")%2==1) + sb.append("'"); + if (truncated) + sb.append("..."); + sb.append('\n'); + } + return true; + } + + + /** see also append(TableInfo, String alias) */ + public SQLFragment append(TableInfo table) + { + SQLFragment s = table.getSQLName(); + if (s != null) + return append(s); + + String alias = table.getSqlDialect().makeLegalIdentifier(table.getName()); + return append(table.getFromSQL(alias)); + } + + /** Add a table/query to the SQL with an alias, as used in a FROM clause */ + public SQLFragment append(TableInfo table, String alias) + { + return append(table.getFromSQL(alias)); + } + + /** Add to the SQL */ + @Override + public SQLFragment append(char ch) + { + getStringBuilder().append(ch); + return this; + } + + /** This is like appendValue(CharSequence s), but force use of literal syntax + * CAUTIONARY NOTE: String literals in PostgresSQL are tricky because of overloaded functions + * array_agg('string') fails array_agg('string'::VARCHAR) works + * json_object('{}) works json_object('string'::VARCHAR) fails + * In the case of json_object() it expects TEXT. Postgres will promote 'json' to TEXT, but not 'json'::VARCHAR + */ + public SQLFragment appendStringLiteral(CharSequence s, @NotNull SqlDialect d) + { + if (null==s) + return appendNull(); + getStringBuilder().append(d.getStringHandler().quoteStringLiteral(s.toString())); + return this; + } + + /** Add to the SQL as either an in-line string literal or as a JDBC parameter depending on whether it would need escaping */ + public SQLFragment appendValue(CharSequence s) + { + return appendValue(s, null); + } + + public SQLFragment appendValue(CharSequence s, SqlDialect d) + { + if (null==s) + return appendNull(); + if (null==d || s.length() > 200) + return append("?").add(s.toString()); + appendStringLiteral(s, d); + return this; + } + + public SQLFragment appendInClause(@NotNull Collection params, SqlDialect dialect) + { + dialect.appendInClauseSql(this, params); + return this; + } + + public CharSequence getSqlCharSequence() + { + if (null != sb) + { + return sb; + } + return sql; + } + + public void insert(int index, SQLFragment sql) + { + if (!sql.getParams().isEmpty()) + { + throw new IllegalArgumentException("Not supported for SQLFragments with parameters - they must be inserted/merged separately"); + } + if (sql.commonTableExpressionsMap != null && !sql.commonTableExpressionsMap.isEmpty()) + { + throw new IllegalArgumentException("Not supported for SQLFragments with CTEs - they must be inserted/merged separately"); + } + if (!tempTokens.isEmpty()) + { + throw new IllegalArgumentException("Not supported for SQLFragments with temp tokens - they must be inserted/merged separately"); + } + getStringBuilder().insert(index, sql.getRawSQL()); + } + + /** Insert into the SQL */ + public void insert(int index, String str) + { + if ((StringUtils.countMatches(str, '\'') % 2) != 0 || + (StringUtils.countMatches(str, '\"') % 2) != 0 || + StringUtils.contains(str, ';')) + { + if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + throw new IllegalArgumentException("SQLFragment.insert(int,String) does not allow semicolons or unmatched quotes"); + } + + getStringBuilder().insert(index, str); + } + + /** Insert this SQLFragment's SQL and parameters at the start of the existing SQL and parameters */ + public void prepend(SQLFragment sql) + { + getStringBuilder().insert(0, sql.getSqlCharSequence().toString()); + if (null != sql.params) + getMutableParams().addAll(0, sql.params); + mergeCommonTableExpressions(sql); + } + + + public int indexOf(String str) + { + return getStringBuilder().indexOf(str); + } + + + // Display query in "English" (display SQL with params substituted) + // with a little more work could probably be made to be SQL legal + public String getFilterText() + { + String sql = getSQL().replaceFirst("WHERE ", ""); + List params = getParams(); + for (Object param1 : params) + { + String param = param1.toString(); + param = param.replaceAll("\\\\", "\\\\\\\\"); + param = param.replaceAll("\\$", "\\\\\\$"); + sql = sql.replaceFirst("\\?", param); + } + return sql.replaceAll("\"", ""); + } + + + @Override + public char charAt(int index) + { + return getSqlCharSequence().charAt(index); + } + + @Override + public int length() + { + return getSqlCharSequence().length(); + } + + @Override + public @NotNull CharSequence subSequence(int start, int end) + { + return getSqlCharSequence().subSequence(start, end); + } + + /** + * KEY is used as a faster way to look for equivalent CTE expressions. + * returning a name here allows us to potentially merge CTE at add time + * + * if you don't have a key you can just use sqlf.toString() + */ + public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf) + { + return addCommonTableExpression(dialect, key, proposedName, sqlf, false); + } + + public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf, boolean recursive) + { + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + CTE prev = commonTableExpressionsMap.get(key); + if (null != prev) + return prev.token(); + CTE cte = new CTE(dialect, proposedName, sqlf, recursive); + commonTableExpressionsMap.put(key, cte); + return cte.token(); + } + + public String createCommonTableExpressionToken(SqlDialect dialect, Object key, String proposedName) + { + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + CTE prev = commonTableExpressionsMap.get(key); + if (null != prev) + throw new IllegalStateException("Cannot create CTE token from already used key."); + CTE cte = new CTE(dialect ,proposedName); + commonTableExpressionsMap.put(key, cte); + return cte.token(); + } + + public void setCommonTableExpressionSql(Object key, SQLFragment sqlf, boolean recursive) + { + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + + if (null != sqlf.commonTableExpressionsMap && !sqlf.commonTableExpressionsMap.isEmpty()) + { + // Need to merge CTEs up; this.cte depends on newSql.ctes, so they need to come first + SQLFragment newSql = new SQLFragment(sqlf); + LinkedHashMap toMap = new LinkedHashMap<>(newSql.commonTableExpressionsMap); + for (Map.Entry e : commonTableExpressionsMap.entrySet()) + { + CTE from = e.getValue(); + CTE to = toMap.get(e.getKey()); + if (null != to) + to.tokens.addAll(from.tokens); + else + toMap.put(e.getKey(), from.copy(false)); + } + + commonTableExpressionsMap = toMap; + newSql.commonTableExpressionsMap = null; + sqlf = newSql; + } + + CTE cte = commonTableExpressionsMap.get(key); + if (null == cte) + throw new IllegalStateException("CTE not found."); + cte.sqlf = sqlf; + cte.recursive = recursive; + } + + + private void mergeCommonTableExpressions(SQLFragment sqlFrom) + { + if (null == sqlFrom.commonTableExpressionsMap || sqlFrom.commonTableExpressionsMap.isEmpty()) + return; + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + for (Map.Entry e : sqlFrom.commonTableExpressionsMap.entrySet()) + { + CTE from = e.getValue(); + CTE to = commonTableExpressionsMap.get(e.getKey()); + if (null != to) + to.tokens.addAll(from.tokens); + else + commonTableExpressionsMap.put(e.getKey(), from.copy(false)); + } + } + + + public void addTempToken(Object tempToken) + { + tempTokens.add(tempToken); + } + + public void addTempTokens(SQLFragment other) + { + tempTokens.add(other.tempTokens); + } + + public static SQLFragment prettyPrint(SQLFragment from) + { + SQLFragment sqlf = new SQLFragment(from); + + String s = from.getSqlCharSequence().toString(); + StringBuilder sb = new StringBuilder(s.length() + 200); + String[] lines = StringUtils.split(s, '\n'); + int indent = 0; + + for (String line : lines) + { + String t = line.trim(); + + if (t.isEmpty()) + continue; + + if (t.startsWith("-- params = b.getParams(); + assertEquals(2,params.size()); + assertEquals(5, params.get(0)); + assertEquals("xxyzzy", params.get(1)); + + + SQLFragment c = new SQLFragment(b); + assertEquals(""" + WITH + /*CTE*/ + \tCTE AS (SELECT a FROM b WHERE x=?) + SELECT * FROM CTE WHERE y=?""", + c.getSQL()); + assertEquals(""" + WITH + /*CTE*/ + \tCTE AS (SELECT a FROM b WHERE x=5) + SELECT * FROM CTE WHERE y='xxyzzy'""", + filterDebugString(c.toDebugString())); + params = c.getParams(); + assertEquals(2,params.size()); + assertEquals(5, params.get(0)); + assertEquals("xxyzzy", params.get(1)); + + + // combining + + SQLFragment sqlf = new SQLFragment(); + String token = sqlf.addCommonTableExpression(dialect, "KEY_A", "cte1", new SQLFragment("SELECT * FROM a")); + sqlf.append("SELECT * FROM ").append(token).append(" _1"); + + assertEquals(""" + WITH + /*CTE*/ + \tcte1 AS (SELECT * FROM a) + SELECT * FROM cte1 _1""", + sqlf.getSQL()); + + SQLFragment sqlf2 = new SQLFragment(); + String token2 = sqlf2.addCommonTableExpression(dialect, "KEY_A", "cte2", new SQLFragment("SELECT * FROM a")); + sqlf2.append("SELECT * FROM ").append(token2).append(" _2"); + assertEquals(""" + WITH + /*CTE*/ + \tcte2 AS (SELECT * FROM a) + SELECT * FROM cte2 _2""", + sqlf2.getSQL()); + + SQLFragment sqlf3 = new SQLFragment(); + String token3 = sqlf3.addCommonTableExpression(dialect, "KEY_B", "cte3", new SQLFragment("SELECT * FROM b")); + sqlf3.append("SELECT * FROM ").append(token3).append(" _3"); + assertEquals(""" + WITH + /*CTE*/ + \tcte3 AS (SELECT * FROM b) + SELECT * FROM cte3 _3""", + sqlf3.getSQL()); + + SQLFragment union = new SQLFragment(); + union.append(sqlf); + union.append("\nUNION\n"); + union.append(sqlf2); + union.append("\nUNION\n"); + union.append(sqlf3); + assertEquals(""" + WITH + /*CTE*/ + \tcte1 AS (SELECT * FROM a) + ,/*CTE*/ + \tcte3 AS (SELECT * FROM b) + SELECT * FROM cte1 _1 + UNION + SELECT * FROM cte1 _2 + UNION + SELECT * FROM cte3 _3""", + union.getSQL()); + } + + @Test + public void nested_cte() + { + // one-level cte using cteToken (CTE fragment 'a' does not contain a CTE) + { + SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); + assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); + SQLFragment b = new SQLFragment(); + String cteToken = b.addCommonTableExpression(dialect, new Object(), "CTE", a); + b.append("SELECT * FROM ").append(cteToken).append(" WHERE p=?").add("parameterTWO"); + assertEquals(""" + WITH + /*CTE*/ + \tCTE AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) + SELECT * FROM CTE WHERE p='parameterTWO'""", + filterDebugString(b.toDebugString())); + assertEquals("parameterONE", b.getParams().get(0)); + } + + // two-level cte using cteTokens (CTE fragment 'b' contains a CTE of fragment a) + { + SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); + assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); + SQLFragment b = new SQLFragment(); + String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); + b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterTWO"); + SQLFragment c = new SQLFragment(); + String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); + c.append("SELECT * FROM ").append(cteTokenB).append(" WHERE i=?").add(3); + assertEquals(""" + WITH + /*CTE*/ + \tA_ AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) + ,/*CTE*/ + \tB_ AS (SELECT * FROM A_ WHERE p='parameterTWO') + SELECT * FROM B_ WHERE i=3""", + filterDebugString(c.toDebugString())); + List params = c.getParams(); + assertEquals(3, params.size()); + assertEquals("parameterONE", params.get(0)); + assertEquals("parameterTWO", params.get(1)); + assertEquals(3, params.get(2)); + } + + // Same as previous but top-level query has both a nested and non-nested CTE + { + SQLFragment a = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); + SQLFragment a2 = new SQLFragment("SELECT 2 as i, 'Atwo' as s, CAST(? AS VARCHAR) as p", "parameterAtwo"); + SQLFragment b = new SQLFragment(); + String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); + b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); + SQLFragment c = new SQLFragment(); + String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); + String cteTokenA2 = c.addCommonTableExpression(dialect, new Object(), "A2_", a2); + c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); + assertEquals(""" + WITH + /*CTE*/ + \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) + ,/*CTE*/ + \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') + ,/*CTE*/ + \tA2_ AS (SELECT 2 as i, 'Atwo' as s, CAST('parameterAtwo' AS VARCHAR) as p) + SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", + filterDebugString(c.toDebugString())); + List params = c.getParams(); + assertEquals(4, params.size()); + assertEquals("parameterAone", params.get(0)); + assertEquals("parameterB", params.get(1)); + assertEquals("parameterAtwo", params.get(2)); + assertEquals(4, params.get(3)); + } + + // Same as previous but two of the CTEs are the same and should be collapsed (e.g. imagine a container filter implemented with a CTE) + // TODO, we only collapse CTEs that are siblings + { + SQLFragment cf = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); + SQLFragment b = new SQLFragment(); + String cteTokenA = b.addCommonTableExpression(dialect, "CTE_KEY_CF", "A_", cf); + b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); + SQLFragment c = new SQLFragment(); + String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); + String cteTokenA2 = c.addCommonTableExpression(dialect, "CTE_KEY_CF", "A2_", cf); + c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); + assertEquals(""" + WITH + /*CTE*/ + \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) + ,/*CTE*/ + \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') + ,/*CTE*/ + \tA2_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) + SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", + filterDebugString(c.toDebugString())); + List params = c.getParams(); + assertEquals(4, params.size()); + assertEquals("parameterAone", params.get(0)); + assertEquals("parameterB", params.get(1)); + assertEquals("parameterAone", params.get(2)); + assertEquals(4, params.get(3)); + } + } + + + private void shouldFail(Runnable r) + { + try + { + r.run(); + if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + fail("Expected IllegalArgumentException"); + } + catch (IllegalArgumentException e) + { + if (AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + fail("Did not expect IllegalArgumentException"); + } + } + + + @Test + public void testIllegalArgument() + { + shouldFail(() -> new SQLFragment(";")); + shouldFail(() -> new SQLFragment().append(";")); + shouldFail(() -> new SQLFragment("AND name='")); + shouldFail(() -> new SQLFragment().append("AND name = '")); + shouldFail(() -> new SQLFragment().append("AND name = 'Robert'); DROP TABLE Students; --")); + + shouldFail(() -> new SQLFragment().appendIdentifier("column name")); + shouldFail(() -> new SQLFragment().appendIdentifier("?")); + shouldFail(() -> new SQLFragment().appendIdentifier(";")); + shouldFail(() -> new SQLFragment().appendIdentifier("\"column\"name\"")); + } + + + String mysqlQuoteIdentifier(String id) + { + return "`" + id.replaceAll("`", "``") + "`"; + } + + @Test + public void testMysql() + { + // OK + new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("mysql")); + new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my`sql")); + new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my\"sql")); + + // not OK + shouldFail(() -> new SQLFragment().appendIdentifier("`")); + shouldFail(() -> new SQLFragment().appendIdentifier("`a`a`")); + } + } + + + public static class IntegrationTestCase extends Assert + { + @Test + public void test() + { + // try some Dialect stuff and CTE executed against core schema + } + } + + + @Override + public boolean equals(Object obj) + { + if (!(obj instanceof SQLFragment other)) + { + return false; + } + return getSQL().equals(other.getSQL()) && getParams().equals(other.getParams()); + } + + /** + * Joins the SQLFragments in the provided {@code Iterable} into a single SQLFragment. The SQL is joined by string + * concatenation using the provided separator. The parameters are combined to form the new parameter list. + * + * @param fragments SQLFragments to join together + * @param separator Separator to use on the SQL portion + * @return A new SQLFragment that joins all the SQLFragments + */ + public static SQLFragment join(Iterable fragments, String separator) + { + if (separator.contains("?")) + throw new IllegalStateException("separator must not include a parameter marker"); + + // Join all the SQL statements + String sql = StreamSupport.stream(fragments.spliterator(), false) + .map(SQLFragment::getSQL) + .collect(Collectors.joining(separator)); + + // Collect all the parameters to a single list + List params = StreamSupport.stream(fragments.spliterator(), false) + .map(SQLFragment::getParams) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + + return new SQLFragment(sql, params); + } + + // Marker interface to hint that this value may be replaced by CURRENT_TIMESTAMP + public static class NowTimestamp extends java.sql.Timestamp + { + public NowTimestamp() + { + this(System.currentTimeMillis()); + } + + public NowTimestamp(long ms) + { + super(ms); + } + } +} diff --git a/api/src/org/labkey/api/data/StatementUtils.java b/api/src/org/labkey/api/data/StatementUtils.java index 3aeb539d8d8..f6ae097188c 100644 --- a/api/src/org/labkey/api/data/StatementUtils.java +++ b/api/src/org/labkey/api/data/StatementUtils.java @@ -1,1916 +1,1917 @@ -/* - * Copyright (c) 2012-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.data; - -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.CaseInsensitiveMapWrapper; -import org.labkey.api.collections.Sets; -import org.labkey.api.data.dialect.MockSqlDialect; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.dataiterator.SimpleTranslator; -import org.labkey.api.dataiterator.TableInsertUpdateDataIterator; -import org.labkey.api.exp.MvColumn; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.list.ListDefinition; -import org.labkey.api.exp.list.ListService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainKind; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.DomainUtil; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.gwt.client.model.GWTDomain; -import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; -import org.labkey.api.query.AliasManager; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryService; -import org.labkey.api.security.User; -import org.labkey.api.util.DateUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JunitUtil; -import org.labkey.api.util.TestContext; -import org.labkey.api.util.logging.LogHelper; - -import java.sql.Connection; -import java.sql.SQLException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Stream; - -import static java.util.Objects.requireNonNull; -import static org.labkey.api.util.JunitUtil.deleteTestContainer; - -public class StatementUtils -{ - private static final Logger _log = LogHelper.getLogger(StatementUtils.class, "SQL insert/update/delete generation"); - - public enum Operation {insert, update, merge} - - // configuration parameters - private Operation _operation; - private SqlDialect _dialect; - private final TableInfo _targetTable; - private Set _keyColumnNames = null; // override the primary key of _table - private Set _skipColumnNames = Set.of(); - private final Set _dontUpdateColumnNames = new CaseInsensitiveHashSet(); - private boolean _updateBuiltInColumns = false; // default to false, this should usually be handled by StandardDataIteratorBuilder - private boolean _selectIds = false; - private boolean _selectObjectUri = false; - private boolean _allowUpdateAutoIncrement = false; - private boolean _preferPKOverObjectUriAsKey = false; - - // variable/parameter tracking helpers - private boolean useVariables = false; - private final Map _constants = new CaseInsensitiveHashMap<>(); - final Map parameters = new CaseInsensitiveMapWrapper<>(new LinkedHashMap<>()); - - // ColumnTracker is used for test instrumentation and contains sets of column names that were included in - // given operations (insert, update, select) after running createStatement(). - // This is intended for test instrumentation use only. - private record ColumnTracker(Set insertColumns, Set updateColumns, Set selectColumns) - { - public ColumnTracker() - { - this(new CaseInsensitiveHashSet(), new CaseInsensitiveHashSet(), new CaseInsensitiveHashSet()); - } - } - - private ColumnTracker _columnTracker; - - // - // builder style methods - // - - //Vocabulary adhoc properties - private Set _vocabularyProperties = new HashSet<>(); - - public StatementUtils(@NotNull Operation op, @NotNull TableInfo table) - { - _operation = op; - _dialect = table.getSqlDialect(); - _targetTable = table; - } - - public StatementUtils dialect(SqlDialect dialect) - { - _dialect = dialect; - return this; - } - - public StatementUtils operation(@NotNull Operation op) - { - _operation = op; - return this; - } - - public StatementUtils constants(@NotNull Map constants) - { - _constants.putAll(constants); - return this; - } - - public StatementUtils keys(Set keyNames) - { - _keyColumnNames = keyNames; - return this; - } - - public StatementUtils skip(Set skip) - { - _skipColumnNames = null==skip ? Set.of() : skip; - return this; - } - - public StatementUtils noupdate(Set noupdate) - { - if (null != noupdate) - _dontUpdateColumnNames.addAll(noupdate); - return this; - } - - public StatementUtils updateBuiltinColumns(boolean b) - { - _updateBuiltInColumns = b; - return this; - } - - public StatementUtils selectIds(boolean b) - { - _selectIds = b; - return this; - } - - public StatementUtils selectObjectUri(boolean b) - { - _selectObjectUri = b; - return this; - } - - public StatementUtils allowSetAutoIncrement(boolean b) - { - _allowUpdateAutoIncrement = b; - return this; - } - - public StatementUtils setVocabularyProperties(Set vocabularyProperties) - { - _vocabularyProperties = vocabularyProperties; - return this; - } - - public StatementUtils setPreferPKOverObjectUriAsKey(boolean preferPKOverObjectUriAsKey) - { - _preferPKOverObjectUriAsKey = preferPKOverObjectUriAsKey; - return this; - } - - private static StatementUtils insertStatement(TableInfo table, boolean selectIds, boolean autoFillDefaultColumns) - { - return new StatementUtils(Operation.insert, table) - .updateBuiltinColumns(autoFillDefaultColumns) - .selectIds(selectIds); - } - - /** - * Create a reusable SQL Statement for inserting rows into a labkey relationship. The relationship - * persisted directly in the database (SchemaTableInfo), or via the OntologyManager tables. - *

- * QueryService shouldn't really know about the internals of exp.Object and exp.ObjectProperty etc. - * However, I can only keep so many levels of abstraction in my head at once. - *

- * NOTE: this is currently fairly expensive for updating one row into an Ontology stored relationship on Postgres. - * This shouldn't be a big problem since we don't usually need to optimize the one-row case, and we're moving - * to provisioned tables for major datatypes. - */ - public static ParameterMapStatement insertStatement(Connection conn, TableInfo table, @Nullable Container c, @Nullable User user, boolean selectIds, boolean autoFillDefaultColumns) throws SQLException - { - return insertStatement(table, selectIds, autoFillDefaultColumns) - .createStatement(conn, c, user); - } - - private static StatementUtils updateStatement(TableInfo table, boolean selectIds, boolean autoFillDefaultColumns) - { - return new StatementUtils(Operation.update, table) - .updateBuiltinColumns(autoFillDefaultColumns) - .selectIds(selectIds); - } - - /** - * Create a reusable SQL Statement for updating rows into a labkey relationship. The relationship - * persisted directly in the database (SchemaTableInfo), or via the OntologyManager tables. - *

- * QueryService shouldn't really know about the internals of exp.Object and exp.ObjectProperty etc. - * However, I can only keep so many levels of abstraction in my head at once. - *

- * NOTE: this is currently fairly expensive for updating one row into an Ontology stored relationship on Postgres. - * This shouldn't be a big problem since we don't usually need to optimize the one-row case, and we're moving - * to provisioned tables for major datatypes. - */ - public static ParameterMapStatement updateStatement(Connection conn, TableInfo table, @Nullable Container c, User user, boolean selectIds, boolean autoFillDefaultColumns) throws SQLException - { - return updateStatement(table, selectIds, autoFillDefaultColumns) - .createStatement(conn, c, user); - } - - private static StatementUtils mergeStatement(TableInfo table, @Nullable Set keyNames, @Nullable Set skipColumnNames, @Nullable Set dontUpdate, boolean selectIds, boolean autoFillDefaultColumns, boolean supportsAutoIncrementKey) - { - return new StatementUtils(Operation.merge, table) - .keys(keyNames) - .skip(skipColumnNames) - .allowSetAutoIncrement(supportsAutoIncrementKey) - .noupdate(dontUpdate) - .updateBuiltinColumns(autoFillDefaultColumns) - .selectIds(selectIds); - } - - public static ParameterMapStatement mergeStatement(Connection conn, TableInfo table, @Nullable Set keyNames, @Nullable Set skipColumnNames, @Nullable Set dontUpdate, @Nullable Container c, @Nullable User user, boolean selectIds, boolean autoFillDefaultColumns, boolean supportsAutoIncrementKey) throws SQLException - { - return mergeStatement(table, keyNames, skipColumnNames, dontUpdate, selectIds, autoFillDefaultColumns, supportsAutoIncrementKey) - .createStatement(conn, c, user); - } - - /* - * Parameter and Variable helpers - */ - - private static class ParameterHolder - { - ParameterHolder(Parameter p) - { - this.p = p; - _columnInfo = null; - } - - ParameterHolder(Parameter p, ColumnInfo c) - { - this.p = p; - _columnInfo = c; - } - - int getScale() - { - var type = requireNonNull(p.getType()); - if (null == _columnInfo || _columnInfo.getScale() <= 0) - return -1; - // GUID.isText()==true - if (JdbcType.GUID != type && (type.isText() || type.isDecimal())) - return _columnInfo.getScale(); - return -1; - } - - int getPrecision() - { - return null==_columnInfo ? -1 : _columnInfo.getPrecision(); - } - - final Parameter p; - final ColumnInfo _columnInfo; - String variableName = null; - Object constantValue = null; - boolean isConstant = false; - } - - private final static String pgRowVarPrefix = "$1."; - private String makeVariableName(String name) - { - String shortName = StringUtils.substring(name,0,32); // name is just for readability, make it short - String uniquePrefix = (_dialect.isSqlServer() ? "@" : pgRowVarPrefix) + ("p" + (parameters.size()+1) + "_"); - return uniquePrefix + AliasManager.makeLegalName(shortName, _dialect, true, uniquePrefix.length()); - } - - private String makePgRowTypeName(String variableName) - { - return StringUtils.substringAfter(variableName, pgRowVarPrefix); - } - - private ParameterHolder createParameter(ColumnInfo c) - { - ParameterHolder ph = parameters.get(c.getName()); - if (null == ph) - { - ph = new ParameterHolder(new Parameter(c.getName(), c.getPropertyURI(), null, c.getJdbcType()), c); - // NOTE: earlier DataIterator should probably split file into two columns: attachment_name, attachment_body - if (c.getInputType().equalsIgnoreCase("file") && c.getJdbcType() == JdbcType.VARCHAR) - ph.p.setFileAsName(true); - initParameterHolder(ph); - parameters.put(c.getName(), ph); - } - return ph; - } - - private void initParameterHolder(ParameterHolder ph) - { - String name = ph.p.getName(); - JdbcType type = ph.p.getType(); - assert null != type; - if (_constants.containsKey(name)) - { - Object value = Parameter.getValueToBind(_constants.get(name), type); - if (null == value || value instanceof Number || value instanceof String || value instanceof java.util.Date) - { - ph.isConstant = true; - ph.constantValue = value; - } - } - ph.variableName = makeVariableName(name); - } - - - private ParameterHolder createParameter(String name, JdbcType type) - { - ParameterHolder ph = parameters.get(name); - if (null == ph) - { - ph = new ParameterHolder(new Parameter(name, type)); - initParameterHolder(ph); - parameters.put(name, ph); - } - return ph; - } - - - private ParameterHolder createParameter(String name, String uri, JdbcType type) - { - ParameterHolder ph = parameters.get(name); - if (null == ph) - { - ph = new ParameterHolder(new Parameter(name, uri, null, type)); - initParameterHolder(ph); - parameters.put(name, ph); - } - return ph; - } - - private SQLFragment appendParameterOrVariable(SQLFragment f, ParameterHolder ph) - { - if (ph.isConstant) - { - toLiteral(f, ph.constantValue); - } - else if (useVariables) - { - f.append(ph.variableName); - } - else - { - f.append("?"); - f.add(ph.p); - } - return f; - } - - private SQLFragment appendPropertyValue(SQLFragment f, DomainProperty dp, ParameterHolder p) - { - if (dp.getJdbcType() == JdbcType.BOOLEAN) - { - f.append("CASE CAST("); - appendParameterOrVariable(f, p); - f.append(" AS ").append(_dialect.getBooleanDataType()).append(")") - .append(" WHEN ").append(_dialect.getBooleanTRUE()).append(" THEN 1.0 ") - .append(" WHEN ").append(_dialect.getBooleanFALSE()).append(" THEN 0.0 ") - .append(" ELSE NULL END"); - return f; - } - else - { - return appendParameterOrVariable(f, p); - } - } - - private void appendSQLFObjectProperty(SQLFragment sqlfObjectProperty, DomainProperty dp, String objectIdVar, String ifTHEN, String ifEND) - { - PropertyType propertyType = dp.getPropertyDescriptor().getPropertyType(); - ParameterHolder v = createParameter(dp.getName(), dp.getPropertyURI(), propertyType.getJdbcType()); - ParameterHolder mv = createParameter(dp.getName()+ MvColumn.MV_INDICATOR_SUFFIX, dp.getPropertyURI() + MvColumn.MV_INDICATOR_SUFFIX, JdbcType.VARCHAR); - sqlfObjectProperty.append("IF ("); - appendPropertyValue(sqlfObjectProperty, dp, v); - sqlfObjectProperty.append(" IS NOT NULL"); - if (dp.isMvEnabled()) - { - sqlfObjectProperty.append(" OR "); - appendParameterOrVariable(sqlfObjectProperty, mv); - sqlfObjectProperty.append(" IS NOT NULL"); - } - sqlfObjectProperty.append(")"); - sqlfObjectProperty.append(ifTHEN); - sqlfObjectProperty.append("INSERT INTO exp.ObjectProperty (objectid, propertyid, typetag, mvindicator, "); - sqlfObjectProperty.append(propertyType.getValueTypeColumn()); - sqlfObjectProperty.append(") VALUES ("); - sqlfObjectProperty.append(objectIdVar); - sqlfObjectProperty.append(",").appendValue(dp.getPropertyId()); - sqlfObjectProperty.append(",").appendStringLiteral(String.valueOf(propertyType.getStorageType()), _dialect); - sqlfObjectProperty.append(","); - if (dp.isMvEnabled()) - appendParameterOrVariable(sqlfObjectProperty, mv); - else - sqlfObjectProperty.append("NULL"); - sqlfObjectProperty.append(","); - appendPropertyValue(sqlfObjectProperty, dp, v); - sqlfObjectProperty.append(")").appendEOS(); - sqlfObjectProperty.append(ifEND); - sqlfObjectProperty.appendEOS(); - } - - private void appendSQLFDeleteObjectProperty(SQLFragment sqlfDelete, String objectIdVar, List domainProperties, Set vocabularyProperties) - { - var properties = null == domainProperties ? vocabularyProperties : domainProperties; - sqlfDelete.append("DELETE FROM exp.ObjectProperty WHERE ObjectId = "); - sqlfDelete.append(objectIdVar); - sqlfDelete.append(" AND PropertyId IN ("); - String separator = ""; - for (DomainProperty property : properties) - { - sqlfDelete.append(separator); - separator = ", "; - sqlfDelete.appendValue(property.getPropertyId()); - } - sqlfDelete.append(")").appendEOS(); - } - - private void setObjectUriPreselect(SQLFragment sqlfPreselectObject, TableInfo table, LinkedHashMap keys, String objectURIVar, String objectURIColumnName, ParameterHolder objecturiParameter) - { - String setKeyword = _dialect.isPostgreSQL() ? "" : "SET "; - if (Operation.merge == _operation || Operation.update == _operation) - { - // this seems overkill actually, but I'm focused on optimizing insert right now (MAB) - sqlfPreselectObject.append(setKeyword).append(objectURIVar).append(" = COALESCE(("); - sqlfPreselectObject.append("SELECT ").appendIdentifier(table.getColumn(objectURIColumnName).getSelectIdentifier()); - sqlfPreselectObject.append(" FROM ").append(table.getSQLName()); - sqlfPreselectObject.append(getPkWhereClause(keys)); - sqlfPreselectObject.append("),"); - appendParameterOrVariable(sqlfPreselectObject, objecturiParameter); - sqlfPreselectObject.append(")").appendEOS(); - - } - else - { - sqlfPreselectObject.append(setKeyword).append(objectURIVar).append(" = "); - appendParameterOrVariable(sqlfPreselectObject, objecturiParameter); - sqlfPreselectObject.appendEOS(); - } - } - - public ParameterMapStatement createStatement(Connection conn, @Nullable Container c, User user) throws SQLException - { - ParameterMapStatement statement = null; - try - { - statement = createStatement(conn, c, user, false); - } - catch (TableInsertUpdateDataIterator.NoUpdatableColumnInDataException e) - { - // ignore error - } - return statement; - } - - public ParameterMapStatement createStatement(Connection conn, @Nullable Container c, User user, boolean checkUpdatableColumns) throws SQLException, TableInsertUpdateDataIterator.NoUpdatableColumnInDataException - { - if (!(_targetTable instanceof UpdateableTableInfo updatable)) - throw new IllegalArgumentException("Table must be an UpdateableTableInfo"); - - TableInfo table = updatable.getSchemaTableInfo(); - - if (table.getTableType() != DatabaseTableType.TABLE) - throw new IllegalArgumentException("Table must be a database table"); - if (null == table.getMetaDataIdentifier()) - throw new IllegalArgumentException("Table must have a metadata identifier"); - - if (Operation.merge == _operation) - { - if (!_dialect.isPostgreSQL() && !_dialect.isSqlServer()) - throw new IllegalArgumentException("Merge is only supported/tested on postgres and sql server"); - } - - useVariables = Operation.merge == _operation; - String ifTHEN = _dialect.isSqlServer() ? " BEGIN " : " THEN "; - String ifEND = _dialect.isSqlServer() ? " END " : " END IF "; - - if (null != c) - { - assert null == _constants.get("container") || c.getId().equals(_constants.get("container")); - if (null == _constants.get("container")) - _constants.put("container", c.getId()); - } - - String objectURIColumnName = updatable.getObjectUriType() == UpdateableTableInfo.ObjectUriType.schemaColumn - ? updatable.getObjectURIColumnName() - : "objecturi"; - ParameterHolder objecturiParameter = null; - if (null != objectURIColumnName) - objecturiParameter = createParameter(objectURIColumnName, JdbcType.VARCHAR); - - // - // Keys for UPDATE or MERGE - // - LinkedHashMap keys = getKeys(updatable, table, objectURIColumnName, _keyColumnNames, _preferPKOverObjectUriAsKey); - - // - // exp.Objects INSERT - // - - SQLFragment sqlfDeclare = new SQLFragment(); - SQLFragment sqlfPreselectObject = new SQLFragment(); - SQLFragment sqlfInsertObject = new SQLFragment(); - SQLFragment sqlfSelectObject = new SQLFragment(); - SQLFragment sqlfObjectProperty = new SQLFragment(); - SQLFragment sqlfDelete = new SQLFragment(); - - Domain domain = updatable.getDomain(); - DomainKind domainKind = updatable.getDomainKind(); - List properties = Collections.emptyList(); - - boolean hasObjectURIColumn = objectURIColumnName != null && table.getColumn(objectURIColumnName) != null; - boolean alwaysInsertExpObject = (hasObjectURIColumn && updatable.isAlwaysInsertExpObject()) && Operation.update != _operation; - if (hasObjectURIColumn) - _dontUpdateColumnNames.add(objectURIColumnName); -// TODO Should we add created and createdby? Or make the caller decide? - if (Operation.update == _operation) - { - _dontUpdateColumnNames.add("Created"); - _dontUpdateColumnNames.add("CreatedBy"); - } - - String objectIdVar = null; - String objectURIVar = null; - boolean objectUriPreselectSet = false; - boolean isMaterializedDomain = null != domain && null != domainKind && StringUtils.isNotEmpty(domainKind.getStorageSchemaName()); - if (alwaysInsertExpObject || (null != domain && !isMaterializedDomain) || !_vocabularyProperties.isEmpty()) - { - properties = (null==domain||isMaterializedDomain) ? Collections.emptyList() : domain.getProperties(); - - if (alwaysInsertExpObject || !properties.isEmpty() || !_vocabularyProperties.isEmpty()) - { - if (!_dialect.isPostgreSQL() && !_dialect.isSqlServer()) - throw new IllegalStateException("Domains are only supported for sql server and postgres"); - - objectIdVar = _dialect.isPostgreSQL() ? "_$objectid$_" : "@_objectid_"; - sqlfDeclare.append("DECLARE ").append(objectIdVar).append(" BIGINT").appendEOS(); - objectURIVar = _dialect.isPostgreSQL() ? "_$objecturi$_" : "@_objecturi_"; - sqlfDeclare.append("DECLARE ").append(objectURIVar).append(" ").append(_dialect.getSqlTypeName(JdbcType.VARCHAR)).append("(300)").appendEOS(); - useVariables |= _dialect.isPostgreSQL(); - - ParameterHolder containerParameter = createParameter("container", JdbcType.GUID); - - // Insert a new row in exp.Object if there isn't already a row for this object - - // Grab the object's ObjectId based on the pk of the base table - if (hasObjectURIColumn || !_vocabularyProperties.isEmpty()) - { - setObjectUriPreselect(sqlfPreselectObject, table, keys, objectURIVar, objectURIColumnName, objecturiParameter); - objectUriPreselectSet = true; - } - - SQLFragment sqlfWhereObjectURI = new SQLFragment(); - sqlfWhereObjectURI.append("(ObjectURI = ").append(objectURIVar).append(")"); - - // In the update case, it's still possible that there isn't a row in exp.Object - there might have been - // no properties in the domain when the row was originally inserted - sqlfInsertObject.append("INSERT INTO exp.Object (container, objecturi, ownerobjectid) "); - sqlfInsertObject.append("SELECT "); - appendParameterOrVariable(sqlfInsertObject, containerParameter); - sqlfInsertObject.append(" AS Container,"); - appendParameterOrVariable(sqlfInsertObject, objecturiParameter); - sqlfInsertObject.append(" AS ObjectURI, "); - Long ownerObjectId = updatable.getOwnerObjectId(); - sqlfInsertObject.append( null == ownerObjectId ? "NULL" : String.valueOf(ownerObjectId) ).append(" AS OwnerObjectId"); - sqlfInsertObject.append(" WHERE NOT EXISTS (SELECT ObjectURI FROM exp.Object WHERE Container = "); - appendParameterOrVariable(sqlfInsertObject, containerParameter); - sqlfInsertObject.append(" AND ").append(sqlfWhereObjectURI).append(")").appendEOS(); - - // re-grab the object's ObjectId, in case it was just inserted - sqlfSelectObject.append(_dialect.isPostgreSQL() ? "" : "SET ").append(objectIdVar).append(" = ("); - sqlfSelectObject.append("SELECT ObjectId FROM exp.Object WHERE Container = "); - appendParameterOrVariable(sqlfSelectObject, containerParameter); - sqlfSelectObject.append(" AND ").append(sqlfWhereObjectURI).append(")").appendEOS(); - - if (Operation.insert != _operation && (!properties.isEmpty() || !_vocabularyProperties.isEmpty())) - { - // Clear out any existing property values for this domain - if (!properties.isEmpty()) - { - appendSQLFDeleteObjectProperty(sqlfDelete, objectIdVar, properties, null); - } - - // Clear out any existing ad hoc property - if (!_vocabularyProperties.isEmpty()) - { - appendSQLFDeleteObjectProperty(sqlfDelete, objectIdVar, null, _vocabularyProperties); - } - } - } - } - - if (_selectObjectUri) - { - if (objectURIVar == null) - { - objectURIVar = _dialect.isPostgreSQL() ? "_$objecturi$_" : "@_objecturi_"; - sqlfDeclare.append("DECLARE ").append(objectURIVar).append(" ").append(_dialect.getSqlTypeName(JdbcType.VARCHAR)).append("(300)").appendEOS(); - } - - if (!objectUriPreselectSet && (hasObjectURIColumn || !_vocabularyProperties.isEmpty())) - { - setObjectUriPreselect(sqlfPreselectObject, table, keys, objectURIVar, objectURIColumnName, objecturiParameter); - } - } - - - // - // BASE TABLE INSERT() - // - - ColumnInfo col; - List cols = new ArrayList<>(); - List values = new ArrayList<>(); - Set done = Sets.newCaseInsensitiveHashSet(); - - if (_updateBuiltInColumns && Operation.update != _operation) - { - col = table.getColumn("Owner"); - if (null != col && null != user) - { - cols.add(col); - values.add(new SQLFragment().appendValue(user.getUserId())); - done.add("Owner"); - } - col = table.getColumn("CreatedBy"); - if (null != col && null != user) - { - cols.add(col); - values.add(new SQLFragment().appendValue(user.getUserId())); - done.add("CreatedBy"); - } - col = table.getColumn("Created"); - if (null != col) - { - cols.add(col); - values.add(new SQLFragment("CURRENT_TIMESTAMP")); // Instead of {fn now()} -- see #27534 - done.add("Created"); - } - } - - ColumnInfo colModifiedBy = table.getColumn("ModifiedBy"); - if (_updateBuiltInColumns && null != colModifiedBy && null != user) - { - cols.add(colModifiedBy); - values.add(new SQLFragment().appendValue(user.getUserId())); - done.add("ModifiedBy"); - } - - ColumnInfo colModified = table.getColumn("Modified"); - if (_updateBuiltInColumns && null != colModified) - { - cols.add(colModified); - values.add(new SQLFragment("CURRENT_TIMESTAMP")); // Instead of {fn now()} -- see #27534 - done.add("Modified"); - } - ColumnInfo colVersion = table.getVersionColumn(); - if (_updateBuiltInColumns && null != colVersion && !done.contains(colVersion.getName())) - { - SQLFragment expr = colVersion.getVersionUpdateExpression(); - if (null != expr) - { - cols.add(colVersion); - values.add(expr); - done.add(colVersion.getName()); - } - } - - String objectIdColumnName = StringUtils.trimToNull(updatable.getObjectIdColumnName()); - ColumnInfo autoIncrementColumn = null; - CaseInsensitiveHashMap remap = updatable.remapSchemaColumns(); - if (null == remap) - remap = CaseInsensitiveHashMap.of(); - - for (ColumnInfo column : table.getColumns()) - { - if (column instanceof WrappedColumn || column.isCalculated()) - continue; - // if we're allowing the caller to set the auto-increment column, then treat like a regular column - if (column.isAutoIncrement() && !_allowUpdateAutoIncrement) - { - autoIncrementColumn = column; - continue; - } - if (column.isVersionColumn() && column != colModified) - continue; - String name = column.getName(); - if (done.contains(name)) - continue; - done.add(name); - ColumnInfo updatableColumn = updatable.getColumn(column.getName()); - if (updatableColumn != null && updatableColumn.hasDbSequence()) - _dontUpdateColumnNames.add(column.getName()); - - SQLFragment valueSQL = new SQLFragment(); - if (column.getName().equalsIgnoreCase(objectIdColumnName)) - { - valueSQL.append(objectIdVar); - } - else if (column.getName().equalsIgnoreCase(updatable.getObjectURIColumnName()) && null != objecturiParameter) - { - appendParameterOrVariable(valueSQL, objecturiParameter); - } - else - { - if (null != _skipColumnNames && _skipColumnNames.contains(Objects.toString(remap.get(name),name))) - continue; - ParameterHolder ph = createParameter(column); - appendParameterOrVariable(valueSQL, ph); - } - cols.add(column); - values.add(valueSQL); - } - - boolean selectAutoIncrement = false; - - assert cols.size() == values.size() : cols.size() + " columns and " + values.size() + " values - should match"; - - // - // INSERT - // - - String comma; - String rowIdVar = null; - SQLFragment sqlfInsertInto = new SQLFragment(); - - // Construct a new column tracker for test instrumentation - _columnTracker = new ColumnTracker(); - - if (Operation.insert == _operation || Operation.merge == _operation) - { - // Create a standard INSERT INTO table (col1, col2) VALUES (val1, val2) statement - // or (for degenerate, empty values case) INSERT INTO table VALUES (DEFAULT) - sqlfInsertInto.append("INSERT INTO ").append(table.getSQLName()); - - if (values.isEmpty()) - { - sqlfInsertInto.append("\nVALUES (DEFAULT)"); - } - else - { - sqlfInsertInto.append(" ("); - comma = ""; - for (ColumnInfo colInfo : cols) - { - sqlfInsertInto.append(comma); - comma = ", "; - sqlfInsertInto.appendIdentifier(colInfo.getSelectIdentifier()); - _columnTracker.insertColumns.add(colInfo.getName()); - } - sqlfInsertInto.append(")"); - - sqlfInsertInto.append("\nSELECT "); - comma = ""; - for (SQLFragment valueSQL : values) - { - sqlfInsertInto.append(comma); - comma = ", "; - sqlfInsertInto.append(valueSQL); - } - } - - if (_selectIds && null != autoIncrementColumn) - { - selectAutoIncrement = true; - if (useVariables) - rowIdVar = "_rowid_"; - rowIdVar = _dialect.addReselect(sqlfInsertInto, autoIncrementColumn, rowIdVar); - if (useVariables) - sqlfDeclare.append("DECLARE ").append(rowIdVar).append(" BIGINT").appendEOS(); // CONSIDER: Move this into addReselect()? - } - - if (_selectObjectUri && hasObjectURIColumn) - { - _dialect.addReselect(sqlfInsertInto, table.getColumn(objectURIColumnName), objectURIVar); - } - } - - // - // UPDATE - // - - SQLFragment sqlfUpdate = new SQLFragment(); - if (Operation.update == _operation || Operation.merge == _operation) - { - // Create a standard UPDATE table SET col1 = val1, col2 = val2 statement - sqlfUpdate.append("UPDATE ").append(table.getSQLName()).append("\nSET "); - comma = ""; - int updateCount = 0; - for (int i = 0; i < cols.size(); i++) - { - col = cols.get(i); - FieldKey fk = col.getFieldKey(); - if (keys.containsKey(fk)) - continue; - - // Issue 52666: Check column remapping when looking for columns to not update - String colName = col.getName(); - if (_dontUpdateColumnNames.contains(colName) || (remap.containsKey(colName) && _dontUpdateColumnNames.contains(remap.get(colName)))) - continue; - - sqlfUpdate.append(comma); - comma = ", "; - sqlfUpdate.appendIdentifier(col.getSelectIdentifier()); - sqlfUpdate.append(" = "); - sqlfUpdate.append(values.get(i)); - _columnTracker.updateColumns.add(col.getName()); - updateCount++; - } - - if (Operation.update == _operation && updateCount == 0) - { - if (checkUpdatableColumns) - throw new TableInsertUpdateDataIterator.NoUpdatableColumnInDataException(table.getName()); - - sqlfUpdate.appendIdentifier(keys.values().iterator().next().getSelectIdentifier()); - sqlfUpdate.append(" = 'noop' WHERE 1 <> 1").appendEOS(); - } - else - { - sqlfUpdate.append(getPkWhereClause(keys)); - sqlfUpdate.appendEOS(); - } - - if (Operation.merge == _operation) - { - // updateCount can equal 0. This happens particularly when inserting into junction tables where - // there are two columns and both are in the primary key - if (0 == updateCount) - { - sqlfUpdate = new SQLFragment(); - sqlfInsertInto.append("\nWHERE NOT EXISTS (SELECT * FROM ").append(table.getSQLName()); - sqlfInsertInto.append(getPkWhereClause(keys)); - sqlfInsertInto.append(")"); - } - else - { - sqlfUpdate.append("IF "); - sqlfUpdate.append(_dialect.isSqlServer() ? "@@ROWCOUNT=0" : "NOT FOUND"); - sqlfUpdate.append(ifTHEN).append("\n\t"); - - sqlfInsertInto.appendEOS(); - sqlfInsertInto.append(ifEND); - } - } - } - - if (Operation.insert == _operation || Operation.merge == _operation) - sqlfInsertInto.appendEOS(); - - SQLFragment sqlfSelectIds = null; - - if ((_selectIds && (null != objectIdVar || null != rowIdVar)) || (_selectObjectUri && null != objectURIVar)) - { - sqlfSelectIds = new SQLFragment("SELECT "); - comma = ""; - if (_selectIds) - { - if (null != rowIdVar) - { - sqlfSelectIds.append(rowIdVar); - _columnTracker.selectColumns.add(rowIdVar); - comma = ","; - } - if (null != objectIdVar) - { - sqlfSelectIds.append(comma).append(objectIdVar); - _columnTracker.selectColumns.add(objectIdVar); - comma = ","; - } - } - - if (_selectObjectUri && null != objectURIVar) - { - sqlfSelectIds.append(comma).append(objectURIVar); - _columnTracker.selectColumns.add(objectIdVar); - } - } - - // - // ObjectProperty - // - - if (!properties.isEmpty()) - { - Set skip = updatable.skipProperties(); - if (null != skip) - done.addAll(skip); - - for (DomainProperty dp : properties) - { - // ignore property that 'wraps' a hard column - if (done.contains(dp.getName())) - continue; - appendSQLFObjectProperty(sqlfObjectProperty, dp, objectIdVar, ifTHEN, ifEND); - } - } - - if (!_vocabularyProperties.isEmpty()) - { - for (DomainProperty vocProp: _vocabularyProperties) - { - appendSQLFObjectProperty(sqlfObjectProperty, vocProp, objectIdVar, ifTHEN, ifEND); - } - } - - // - // PREPARE - // - - ParameterMapStatement ret; - - if (!useVariables) - { - SQLFragment script = new SQLFragment(); - Stream.of(sqlfDeclare, sqlfPreselectObject, sqlfInsertObject, sqlfSelectObject, sqlfDelete, sqlfUpdate, sqlfInsertInto, sqlfObjectProperty, sqlfSelectIds) - .filter(f -> null != f && !f.isEmpty()) - .forEach(script::append); - ret = new ParameterMapStatement(table.getSchema().getScope(), conn, script, remap); - } - else if (_dialect.isSqlServer()) - { - if (!parameters.isEmpty()) - { - SQLFragment select = new SQLFragment(); - sqlfDeclare.append("DECLARE "); - select.append("SELECT "); - comma = ""; - for (Map.Entry e : parameters.entrySet()) - { - ParameterHolder ph = e.getValue(); - sqlfDeclare.append(comma); - String variable = sqlServerVariableDeclaration(sqlfDeclare, ph); - select.append(comma).append(variable).append("=?"); - select.add(ph.p); - comma = ", "; - } - sqlfDeclare.appendEOS(); - sqlfDeclare.append(select); - sqlfDeclare.appendEOS(); - } - SQLFragment script = new SQLFragment(); - Stream.of(sqlfDeclare, sqlfPreselectObject, sqlfInsertObject, sqlfSelectObject, sqlfDelete, sqlfUpdate, sqlfInsertInto, sqlfObjectProperty, sqlfSelectIds) - .filter(f -> null != f && !f.isEmpty()) - .forEach(script::append); - _log.debug(script.toDebugString()); - ret = new ParameterMapStatement(table.getSchema().getScope(), conn, script, remap); - } - else - { - // wrap in a function with a single ROW() constructor argument - SQLFragment fn = new SQLFragment(); - String fnName = _dialect.getGlobalTempTablePrefix() + "fn_" + GUID.makeHash(); - TempTableTracker.track(fnName, fn); - - String typeName = fnName + "type"; - fn.append("CREATE TYPE ").append(typeName).append(" AS ("); - // TODO d.execute() doesn't handle temp schema - SQLFragment call = new SQLFragment(); - call.append(fnName).append("(ROW("); - comma = ""; - for (Map.Entry e : parameters.entrySet()) - { - ParameterHolder ph = e.getValue(); - String type = _dialect.getSqlTypeName(ph.p.getType()); - fn.append("\n").append(comma); - fn.append(makePgRowTypeName(ph.variableName)); - fn.append(" "); - fn.append(type); - // For PG (29687) we need the length for CHAR type - if (_dialect.isPostgreSQL() && JdbcType.CHAR.equals(ph.p.getType())) - fn.append("(").appendValue(ph.getScale()).append(")"); - call.append(comma).append("?"); - call.add(ph.p); - comma = ","; - } - fn.append("\n)").appendEOS(); - fn.append("CREATE FUNCTION ").append(fnName).append("(").append(typeName).append(") "); - fn.append("RETURNS "); - if (null != sqlfSelectIds) - fn.append("SETOF RECORD"); - else - fn.append("void"); - String quoteToken = "$x" + GUID.makeHash() + "$"; - fn.append(" AS ").append(quoteToken).append("\n"); - call.append("))"); - - if (null != sqlfSelectIds) - { - call.insert(0, "SELECT * FROM "); - call.append(" AS x("); - String sep = ""; - - if (_selectIds) - { - if (null != rowIdVar) - { - call.append("A BIGINT"); - sep = ", "; - } - if (null != objectIdVar) - { - call.append(sep); - call.append("B BIGINT"); - sep = ", "; - } - } - - if (_selectObjectUri && null != objectURIVar) - { - call.append(sep); - call.append("C VARCHAR"); - } - - call.append(")").appendEOS(); - } - else - { - call.insert(0, "{call "); - call.append("}"); - } - - fn.append(sqlfDeclare); - - fn.append("BEGIN\n"); - fn.append("-- ").append(_operation.name()).append("\n"); - Stream.of(sqlfPreselectObject, sqlfInsertObject, sqlfSelectObject, sqlfDelete, sqlfUpdate, sqlfInsertInto, sqlfObjectProperty) - .filter(f -> null != f && !f.isEmpty()) - .forEach(fn::append); - if (null == sqlfSelectIds) - { - fn.append("RETURN").appendEOS(); - } - else - { - sqlfSelectIds.insert(0, "RETURN QUERY\n"); - fn.append(sqlfSelectIds); - fn.appendEOS(); - } - fn.append("END").appendEOS().append(" ").append(quoteToken).append(" LANGUAGE plpgsql").appendEOS(); - _log.debug(fn.toDebugString()); - _log.debug(call.toDebugString()); - final SQLFragment drop = new SQLFragment("DROP TYPE IF EXISTS ").append(typeName).append(" CASCADE").appendEOS(); - _log.debug(drop.toDebugString()); - new SqlExecutor(table.getSchema()).execute(fn); - ret = new ParameterMapStatement(table.getSchema().getScope(), conn, call, updatable.remapSchemaColumns()); - ret.setDebugSql(fn.getSQL() + "--\n" + call.toDebugString()); - ret.onClose(() -> { - try - { - new SqlExecutor(ExperimentService.get().getSchema()).execute(drop); - } - catch (Exception x) - { - _log.error("Error dropping custom rowtype for temp function.", x); - } - }); - } - - int selectIndex = 1; - - if (_selectIds) - { - // Why is one of these boolean and the other an index?? I don't know - ret.setSelectRowId(selectAutoIncrement); - - if (selectAutoIncrement) - selectIndex++; - - if (null != objectIdVar) - ret.setObjectIdIndex(selectIndex++); - } - - if (_selectObjectUri && null != objectURIVar) - ret.setObjectUriIndex(selectIndex); - - return ret; - } - - private static LinkedHashMap getKeys( - UpdateableTableInfo updatable, - TableInfo table, - String objectURIColumnName, - Set keyColumnNames, - boolean preferPKOverObjectUriAsKey - ) - { - LinkedHashMap keys = new LinkedHashMap<>(); - ColumnInfo col = table.getColumn("Container"); - - if (null != col) - keys.put(col.getFieldKey(), col); - - if (null != keyColumnNames && !keyColumnNames.isEmpty()) - { - for (String name : keyColumnNames) - { - col = table.getColumn(name); - if (null == col) - throw new IllegalArgumentException("Column not found: " + name); - keys.put(col.getFieldKey(), col); - } - } - else - { - // using objectURIColumnName preferentially to be backward compatible with OntologyManager.saveTabDelimited - // which in turn is only called by LuminexDataHandler.saveDataRows() - col = objectURIColumnName == null ? null : table.getColumn(objectURIColumnName); - if (null != col && !preferPKOverObjectUriAsKey) - keys.put(col.getFieldKey(), col); - else - { - // See Issue 26661 and Issue 41053 - // NOTE: IMO we should not be using updatable.getPkColumnNames() here! If the caller doesn't want to use the - // 'real' PK from the SchemaTableInfo for update/merge, then the alternate keys should be explicitly specified - // using StatementUtils.keys() - for (String pkName : updatable.getPkColumnNames()) - { - col = table.getColumn(pkName); - if (null == col) - throw new IllegalStateException("pk column not found: " + pkName); - keys.put(col.getFieldKey(), col); - } - } - } - - return keys; - } - - private SQLFragment getPkWhereClause(LinkedHashMap keys) - { - SQLFragment sqlfWherePK = new SQLFragment(); - sqlfWherePK.append("\nWHERE "); - String and = ""; - for (Map.Entry e : keys.entrySet()) - { - ColumnInfo keyCol = e.getValue(); - ParameterHolder keyColPh = createParameter(keyCol); - - sqlfWherePK.append(and); - sqlfWherePK.append("("); - sqlfWherePK.appendIdentifier(keyCol.getSelectIdentifier()); - sqlfWherePK.append(" = "); - appendParameterOrVariable(sqlfWherePK, keyColPh); - if (keyCol.isNullable()) - { - sqlfWherePK.append(" OR "); - sqlfWherePK.appendIdentifier(keyCol.getSelectIdentifier()); - sqlfWherePK.append(" IS NULL AND "); - appendParameterOrVariable(sqlfWherePK, keyColPh); - sqlfWherePK.append(" IS NULL"); - } - sqlfWherePK.append(")"); - and = " AND "; - } - return sqlfWherePK; - } - - private String sqlServerVariableDeclaration(SQLFragment sqlfDeclare, ParameterHolder ph) - { - assert(_dialect.isSqlServer()); - String variable = ph.variableName; - sqlfDeclare.append(variable); - sqlfDeclare.append(" "); - JdbcType jdbcType = ph.p.getType(); - assert null != jdbcType; - String type = _dialect.getSqlTypeName(jdbcType); - assert null != type; - - // Workaround - SQLServer doesn't support TEXT, NTEXT, or IMAGE as local variables in statements, but is OK with NVARCHAR(MAX) - if (jdbcType.isText()) - { - if ("NTEXT".equalsIgnoreCase(type) || "TEXT".equalsIgnoreCase(type) || ph.getScale()>4000) - type = "NVARCHAR(MAX)"; - else - type = "NVARCHAR(4000)"; - } - // Add scale and precision for decimal values specifying scale - else if (jdbcType.isDecimal() && ph.getScale() > 0) - { - type = type + "(" + ph.getPrecision() + "," + ph.getScale() + ")"; - } - - sqlfDeclare.append(type); - return variable; - } - - /* - * We could use SQLFragment.appendValue() for most of these. However, here it is important to force - * the use of inline literal values. SQLFragment.appendValue() does not guarantee that. - */ - private void toLiteral(SQLFragment f, Object value) - { - if (null == value) - { - f.append("NULL"); - return; - } - if (value instanceof Number) - { - f.append(value.toString()); - return; - } - if (value instanceof SimpleTranslator.NowTimestamp) - { - f.append("CURRENT_TIMESTAMP"); // Instead of {fn now()} -- see #27534 - return; - } - if (value instanceof java.sql.Date sqlDate) - { - f.append("{d ").append(_dialect.getStringHandler().quoteStringLiteral(DateUtil.formatIsoDate(sqlDate))).append("}"); - return; - } - else if (value instanceof java.util.Date date) - { - f.append("{ts ").append(_dialect.getStringHandler().quoteStringLiteral(DateUtil.formatIsoDateShortTime(date))).append("}"); - return; - } - assert value instanceof String; - f.append(_dialect.getStringHandler().quoteStringLiteral(String.valueOf(value))); - } - - @SuppressWarnings("JUnitMalformedDeclaration") - public static class TestCase extends Assert - { - final static String DATA_CLASS_NAME = "StatementUtilsTestDataClass"; - final static String VOCAB_DOMAIN_KIND = "Vocabulary"; // VocabularyDomainKind.KIND_NAME - final static String VOCAB_DOMAIN_NAME = "StatementUtilsVocabularyDomain"; - - final Container container; - final TableInfo dataClassTable; - final TableInfo principalsTable; - final UpdateableTableInfo testTable; - final User user; - final Set vocabParameters = CaseInsensitiveHashSet.of("Age", "AgeMVIndicator", "Color", "ColorMVIndicator"); - final Set vocabProps; - - // Flag to run tests against one or both (Postgres, SQL Server) SqlDialects. Set to false by default - // since tests are run in both environments in CI. See "otherSqlDialect". - final boolean runOtherDialect = false; - - // This is a mock SqlDialect that mocks the alternative SqlDialect configuration to the current configuration. - // So, if the tests are running in a Postgres environment, then this represents a SQL Server SqlDialect - // and vice versa. This is useful for getting code coverage across code paths for both dialects in a single - // test run. Enabled via the "runOtherDialect" flag. - final SqlDialect otherSqlDialect; - - public TestCase() throws Exception - { - container = JunitUtil.getTestContainer(); - user = TestContext.get().getUser(); - - dataClassTable = QueryService.get().getUserSchema(user, container, ExpSchema.SCHEMA_EXP_DATA).getTableOrThrow(DATA_CLASS_NAME); - principalsTable = DbSchema.get("core", DbSchemaType.Module).getTable("principals"); - testTable = DbSchema.get("test", DbSchemaType.Module).getTable("testtable2"); - - // Initialize vocab domain properties - { - var vocabDomainKind = PropertyService.get().getDomainKindByName(VOCAB_DOMAIN_KIND); - var vocabDomainURI = vocabDomainKind.generateDomainURI(null, VOCAB_DOMAIN_NAME, container, user); - var vocabDomain = PropertyService.get().getDomain(container, vocabDomainURI); - assertNotNull(vocabDomain); - vocabProps = Set.of(vocabDomain.getPropertyByName("Age"), vocabDomain.getPropertyByName("Color")); - } - - if (runOtherDialect) - { - SqlDialect defaultDialect = principalsTable.getSqlDialect(); - boolean isPostgres = defaultDialect.isPostgreSQL(); - - otherSqlDialect = new MockSqlDialect() - { - @Override - public String addReselect(SQLFragment sql, ColumnInfo column, @Nullable String proposedVariable) - { - return defaultDialect.addReselect(sql, column, proposedVariable); - } - - @Override - public String getGuidType() - { - return defaultDialect.getGuidType(); - } - - @Override - public @Nullable String getSqlTypeName(JdbcType type) - { - return defaultDialect.getSqlTypeName(type); - } - - @Override - public boolean isPostgreSQL() - { - // Returns true in SQL Server configured environments - return !isPostgres; - } - - @Override - public boolean isSqlServer() - { - // Returns true in Postgres configured environments - return isPostgres; - } - }; - } - else - { - otherSqlDialect = null; - } - } - - @BeforeClass - public static void createDomains() throws Exception - { - var container = JunitUtil.getTestContainer(); - var user = TestContext.get().getUser(); - - // Create a data class domain - ExperimentService.get().createDataClass(container, user, DATA_CLASS_NAME, null, List.of(new GWTPropertyDescriptor("aa", "int")), List.of(), null, null); - - // Create a vocabulary domain - { - GWTPropertyDescriptor prop1 = new GWTPropertyDescriptor(); - prop1.setRangeURI("int"); - prop1.setName("Age"); - prop1.setMvEnabled(true); - - GWTPropertyDescriptor prop2 = new GWTPropertyDescriptor(); - prop2.setRangeURI("string"); - prop2.setName("Color"); - - GWTDomain domain = new GWTDomain<>(); - domain.setName(VOCAB_DOMAIN_NAME); - domain.setFields(List.of(prop1, prop2)); - - DomainUtil.createDomain(VOCAB_DOMAIN_KIND, domain, null, container, user, VOCAB_DOMAIN_NAME, null, false); - } - } - - @AfterClass - public static void cleanup() - { - deleteTestContainer(); - } - - @Test - public void testToLiteral() - { - boolean isPostgres = principalsTable.getSqlDialect().isPostgreSQL(); - - var statement = new StatementUtils(Operation.insert, principalsTable); - Function runToLiteral = (value) -> { - var sql = new SQLFragment(); - statement.toLiteral(sql, value); - return sql; - }; - - var dateLong = 1749759500016L; // Thu Jun 12 2025 13:18:20 GMT-0700 (Pacific Daylight Time) - - // null value - var actual = runToLiteral.apply(null); - assertEquals(new SQLFragment("NULL"), actual); - - // Number - actual = runToLiteral.apply(1234567890); - assertEquals(new SQLFragment("1234567890"), actual); - - // NowTimestamp - actual = runToLiteral.apply(new SimpleTranslator.NowTimestamp(dateLong)); - assertEquals(new SQLFragment("CURRENT_TIMESTAMP"), actual); - - // sql.Date - var sqlDate = new java.sql.Date(dateLong); - var dateFormat = new SimpleDateFormat(DateUtil.getStandardDateFormatString()); - var expected = String.format(isPostgres ? "{d '%s'}" : "{d N'%s'}", dateFormat.format(sqlDate)); - - actual = runToLiteral.apply(sqlDate); - assertEquals(new SQLFragment(expected), actual); - - // util.Date - var utilDate = new java.util.Date(dateLong); - dateFormat = new SimpleDateFormat(DateUtil.getStandardDateTimeFormatString()); - expected = String.format(isPostgres ? "{ts '%s'}" : "{ts N'%s'}", dateFormat.format(utilDate)); - - actual = runToLiteral.apply(utilDate); - assertEquals(new SQLFragment(expected), actual); - } - - @Test - public void testCreateStatementValidation() throws Exception - { - try (var conn = getConnection()) - { - var nonUpdateTable = new VirtualTable<>(DbSchema.get("test", DbSchemaType.Module), "virtualInsanity", null); - - var exception = Assert.assertThrows(IllegalArgumentException.class, () -> new StatementUtils(Operation.merge, nonUpdateTable).createStatement(conn, container, user)); - assertEquals("Table must be an UpdateableTableInfo", exception.getMessage()); - - // Unreachable with current mocks -// var nonDatabaseTable = QueryService.get().getUserSchema(user, container, "core").getTableOrThrow("Principals"); -// exception = Assert.assertThrows(IllegalArgumentException.class, () -> new StatementUtils(Operation.merge, nonDatabaseTable).createStatement(conn, container, user)); -// assertEquals("Table must be a database table", exception.getMessage()); - -// exception = Assert.assertThrows(IllegalArgumentException.class, () -> { -// var noIdentifierTable = principalsTable.getMetaDataIdentifier(). -// new StatementUtils(Operation.merge, nonDatabaseTable).dialect(new MockSqlDialect()).createStatement(conn, container, user); -// }); -// assertEquals("Table must have a metadata identifier", exception.getMessage()); - - exception = Assert.assertThrows(IllegalArgumentException.class, () -> new StatementUtils(Operation.merge, principalsTable).dialect(new MockSqlDialect()).createStatement(conn, container, user)); - assertEquals("Merge is only supported/tested on postgres and sql server", exception.getMessage()); - } - } - - @Test - public void testGetKeys() - { - var containerFieldKey = FieldKey.fromParts("Container"); - var rowIdFieldKey = FieldKey.fromParts("RowId"); - var textFieldKey = FieldKey.fromParts("Text"); - - var updateTable = testTable; - var table = updateTable.getSchemaTableInfo(); - - // Pre-conditions - var pkColumnNames = new CaseInsensitiveHashSet(testTable.getPkColumnNames()); - assertEquals(2, pkColumnNames.size()); - assertTrue(pkColumnNames.contains(containerFieldKey.getName())); - assertTrue(pkColumnNames.contains(textFieldKey.getName())); - assertNotNull(testTable.getColumn(rowIdFieldKey)); - - // The "Container" column is always resolved if present on the table - var keys = StatementUtils.getKeys(updateTable, table, null, Set.of(containerFieldKey.getName()), false); - assertEquals(1, keys.size()); - assertTrue(keys.containsKey(containerFieldKey)); - - // The "Container" column is only resolved even when in the explicit name map - keys = StatementUtils.getKeys(updateTable, table, null, Set.of(containerFieldKey.getName()), false); - assertEquals(1, keys.size()); - assertTrue(keys.containsKey(containerFieldKey)); - - // The "Container" column is also resolved even when not in the explicit key column map. Other key columns are included as well. - keys = StatementUtils.getKeys(updateTable, table, null, Set.of(textFieldKey.getName()), false); - assertEquals(2, keys.size()); - assertTrue(keys.containsKey(containerFieldKey)); - assertTrue(keys.containsKey(textFieldKey)); - - // All explicitly named columns should resolve as columns on the table - var exception = Assert.assertThrows(IllegalArgumentException.class, () -> StatementUtils.getKeys(updateTable, table, null, Set.of(textFieldKey.getName(), "Beep"), false)); - assertEquals("Column not found: Beep", exception.getMessage()); - - // Furnish an explicit "objectURIColumnName" and expect it to be included when preferPKOverObjectUriAsKey = false - keys = StatementUtils.getKeys(updateTable, table, "RowId", null, false); - assertEquals(2, keys.size()); - assertTrue(keys.containsKey(containerFieldKey)); - assertTrue(keys.containsKey(rowIdFieldKey)); - - // Furnish an explicit "objectURIColumnName" and expect it to NOT be included when preferPKOverObjectUriAsKey = true - keys = StatementUtils.getKeys(updateTable, table, "RowId", null, true); - assertEquals(2, keys.size()); - assertTrue(keys.containsKey(containerFieldKey)); - assertTrue(keys.containsKey(textFieldKey)); - - keys = StatementUtils.getKeys(updateTable, table, null, null, false); - assertEquals(2, keys.size()); - assertTrue(keys.containsKey(containerFieldKey)); - assertTrue(keys.containsKey(textFieldKey)); - } - - @Test - public void testInsert() throws Exception - { - ParameterMapStatement m = null; - try (Connection conn = getConnection()) - { - m = insertStatement(conn, principalsTable, container, user, true, true); - m.close(); m = null; - - m = insertStatement(conn, testTable, container, user, true, true); - m.close(); m = null; - } - finally - { - if (null != m) - m.close(); - } - } - - @Test - public void testInsertWithExtensibleDomain() throws Exception - { - ParameterMapStatement m = null; - try (Connection conn = getConnection(dataClassTable)) - { - StatementUtils statement; - - // Insert - { - var validateInsert = new Function() - { - @Override - public Object apply(StatementUtils s) - { - boolean isPostgres = s._dialect.isPostgreSQL(); - - assertTrue(s._columnTracker.insertColumns.contains("Created")); - assertTrue(s._columnTracker.insertColumns.contains("CreatedBy")); - assertTrue(s._columnTracker.insertColumns.contains("Modified")); - assertTrue(s._columnTracker.insertColumns.contains("ModifiedBy")); - assertTrue(s._columnTracker.updateColumns.isEmpty()); - - if (isPostgres) - { - assertTrue(s._columnTracker.selectColumns.contains("_rowid_")); - assertTrue(s._columnTracker.selectColumns.contains("_$objectid$_")); - } - else - { - // Variables are not used in SQL Server - assertFalse(s._columnTracker.selectColumns.contains("_rowid_")); - assertTrue(s._columnTracker.selectColumns.contains("@_objectid_")); - } - - var parameterKeys = s.parameters.keySet(); - assertTrue(vocabParameters.stream().noneMatch(parameterKeys::contains)); - return null; - } - }; - - statement = insertStatement(dataClassTable, true, true); - m = statement.createStatement(conn, container, user); - m.close(); - m = null; - validateInsert.apply(statement); - - if (runOtherDialect) - { - statement = insertStatement(dataClassTable, true, true); - statement.dialect(otherSqlDialect); - m = statement.createStatement(conn, container, user); - m.close(); - m = null; - validateInsert.apply(statement); - } - } - - // Insert with vocabulary properties - { - statement = insertStatement(dataClassTable, true, true); - statement.setVocabularyProperties(vocabProps); - m = statement.createStatement(conn, container, user); - m.close(); - m = null; - assertTrue(statement.parameters.keySet().containsAll(vocabParameters)); - } - } - finally - { - if (null != m) - m.close(); - } - } - - @Test - public void testUpdate() throws Exception - { - ParameterMapStatement m = null; - try (Connection conn = getConnection()) - { - m = updateStatement(conn, principalsTable, container, user, true, true); - m.close(); m = null; - - m = updateStatement(conn, testTable, container, user, true, true); - m.close(); m = null; - } - finally - { - if (null != m) - m.close(); - } - } - - @Test - public void testUpdateWithExtensibleDomain() throws Exception - { - ParameterMapStatement m = null; - try (Connection conn = getConnection(dataClassTable)) - { - StatementUtils statement; - - // Update - { - var validateUpdate = new Function() - { - @Override - public Object apply(StatementUtils s) - { - assertTrue(s._columnTracker.insertColumns.isEmpty()); - assertFalse(s._columnTracker.updateColumns.contains("Created")); - assertFalse(s._columnTracker.updateColumns.contains("CreatedBy")); - assertTrue(s._columnTracker.updateColumns.contains("Modified")); - assertTrue(s._columnTracker.updateColumns.contains("ModifiedBy")); - assertTrue(s._columnTracker.selectColumns.isEmpty()); - var parameterKeys = s.parameters.keySet(); - assertTrue(vocabParameters.stream().noneMatch(parameterKeys::contains)); - return null; - } - }; - - statement = updateStatement(dataClassTable, true, true); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - validateUpdate.apply(statement); - - Set allUpdateColumns = new CaseInsensitiveHashSet(statement._columnTracker.updateColumns); - - if (runOtherDialect) - { - statement = updateStatement(dataClassTable, true, true); - statement.dialect(otherSqlDialect); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - validateUpdate.apply(statement); - } - - statement = updateStatement(dataClassTable, false, false); - statement.noupdate(CaseInsensitiveHashSet.of("Modified", "ModifiedBy")); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - assertFalse(statement._columnTracker.updateColumns.contains("Modified")); - assertFalse(statement._columnTracker.updateColumns.contains("ModifiedBy")); - - statement = updateStatement(dataClassTable, false, false); - statement.noupdate(allUpdateColumns); - m = statement.createStatement(conn, container, user); - var debugSql = m.getDebugSql(); - m.close(); m = null; - assertTrue(debugSql.contains("'noop' WHERE 1 <> 1")); - } - - // Update with vocabulary properties - { - statement = updateStatement(dataClassTable, true, true); - statement.setVocabularyProperties(vocabProps); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - assertTrue(statement.parameters.keySet().containsAll(vocabParameters)); - } - } - finally - { - if (null != m) - m.close(); - } - } - - @Test - public void testUpdateWithObjectUriColumn() throws Exception - { - // Arrange - // Create a list - String listName = "StatementUtilsTestList"; - { - // Create a list domain - var listDef = ListService.get().createList(container, listName, ListDefinition.KeyType.AutoIncrementInteger); - listDef.setKeyName("pk"); - - Domain domain = requireNonNull(listDef.getDomain()); - addProperty(domain, "pk", PropertyType.INTEGER); - addProperty(domain, "name", PropertyType.STRING); - - listDef.save(user); - } - - ParameterMapStatement m = null; - TableInfo listTable = requireNonNull(ListService.get().getList(container, listName)).getTable(user, container); - assertNotNull(listTable); - - try (Connection conn = getConnection(listTable)) - { - StatementUtils statement; - var expectedNoUpdateColumns = CaseInsensitiveHashSet.of("EntityId", "Created", "CreatedBy"); - var expectedUpdateColumns = CaseInsensitiveHashSet.of("DIImportHash", "LastIndexed", "Modified", "ModifiedBy", "Name"); - - // Update statement (selectIds = true, autoFillDefaultColumns = true) - { - statement = StatementUtils.updateStatement(listTable, true, true); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - - // Assert - assertEquals(expectedNoUpdateColumns, statement._dontUpdateColumnNames); - assertTrue(statement._columnTracker.insertColumns.isEmpty()); - assertTrue(statement._columnTracker.selectColumns.isEmpty()); - - if (runOtherDialect) - { - statement = StatementUtils.updateStatement(listTable, true, true); - statement.dialect(otherSqlDialect); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - } - - assertEquals(expectedNoUpdateColumns, statement._dontUpdateColumnNames); - assertTrue(statement._columnTracker.insertColumns.isEmpty()); - assertTrue(statement._columnTracker.selectColumns.isEmpty()); - - assertEquals(expectedUpdateColumns, statement._columnTracker.updateColumns); - } - - // Update statement (selectIds = false, autoFillDefaultColumns = false) - { - statement = StatementUtils.updateStatement(listTable, false, false); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - - // Assert - assertEquals(expectedNoUpdateColumns, statement._dontUpdateColumnNames); - assertTrue(statement._columnTracker.insertColumns.isEmpty()); - assertEquals(expectedUpdateColumns, statement._columnTracker.updateColumns); - - if (runOtherDialect) - { - statement = StatementUtils.updateStatement(listTable, false, false); - statement.dialect(otherSqlDialect); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - } - - assertEquals(expectedNoUpdateColumns, statement._dontUpdateColumnNames); - assertTrue(statement._columnTracker.insertColumns.isEmpty()); - assertEquals(expectedUpdateColumns, statement._columnTracker.updateColumns); - } - } - finally - { - if (null != m) - m.close(); - } - } - - @Test - public void testMerge() throws Exception - { - ParameterMapStatement m = null; - try (Connection conn = getConnection()) - { - m = mergeStatement(conn, principalsTable, null, null, null, container, user, false, true, false); - m.close(); m = null; - - if (runOtherDialect) - { - StatementUtils statement = mergeStatement(principalsTable, null, null, null, false, true, false); - statement.dialect(otherSqlDialect); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - } - - m = mergeStatement(conn, testTable, null, null, null, container, user, false, true, false); - m.close(); m = null; - - if (runOtherDialect) - { - StatementUtils statement = mergeStatement(testTable, null, null, null, false, true, false); - statement.dialect(otherSqlDialect); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - } - } - finally - { - if (null != m) - m.close(); - } - } - - @Test - public void testMergeWithExtensibleDomain() throws Exception - { - ParameterMapStatement m = null; - try (Connection conn = getConnection(dataClassTable)) - { - StatementUtils statement; - - // Merge - { - var validateMerge = new Function() - { - @Override - public Object apply(StatementUtils s) - { - boolean isPostgres = s._dialect.isPostgreSQL(); - - assertTrue(s._columnTracker.insertColumns.contains("Container")); - assertTrue(s._columnTracker.insertColumns.contains("LSID")); - assertFalse(s._columnTracker.updateColumns.contains("Container")); - assertFalse(s._columnTracker.updateColumns.contains("LSID")); - - if (isPostgres) - { - assertTrue(s._columnTracker.selectColumns.contains("_rowid_")); - assertTrue(s._columnTracker.selectColumns.contains("_$objectid$_")); - } - else - { - // Variables are not used in SQL Server - assertFalse(s._columnTracker.selectColumns.contains("_rowid_")); - assertTrue(s._columnTracker.selectColumns.contains("@_objectid_")); - } - - var parameterKeys = s.parameters.keySet(); - assertTrue(vocabParameters.stream().noneMatch(parameterKeys::contains)); - return null; - } - }; - - statement = mergeStatement(dataClassTable, null, null, null, true, true, false); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - validateMerge.apply(statement); - - var updateColumns = new CaseInsensitiveHashSet(statement._columnTracker.updateColumns); - - if (runOtherDialect) - { - statement = mergeStatement(dataClassTable, null, null, null, true, true, false); - statement.dialect(otherSqlDialect); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - validateMerge.apply(statement); - } - - // TODO: This generates a SQL parsing error in Postgres due to the reselect statement coming before the WHERE clause -// statement = mergeStatement(dataClassTable, null, CaseInsensitiveHashSet.of("RunId"), updateColumns, true, true, false); -// m = statement.createStatement(conn, container, user); -// m.close(); m = null; -// assertTrue(statement._columnTracker.updateColumns.isEmpty()); - } - - // Merge with vocabulary properties - { - statement = mergeStatement(dataClassTable, null, null, null, false, true, false); - statement.setVocabularyProperties(vocabProps); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - assertTrue(statement.parameters.keySet().containsAll(vocabParameters)); - } - } - finally - { - if (null != m) - m.close(); - } - } - - private static void addProperty(Domain d, String name, PropertyType pt) - { - DomainProperty p = d.addProperty(); - p.setName(name); - p.setPropertyURI(d.getTypeURI() + "#" + name); - p.setRangeURI(pt.getTypeUri()); - } - - private Connection getConnection() throws SQLException - { - return getConnection(principalsTable); - } - - private Connection getConnection(TableInfo table) throws SQLException - { - return table.getSchema().getScope().getConnection(); - } - } -} +/* + * Copyright (c) 2012-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.data; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.CaseInsensitiveMapWrapper; +import org.labkey.api.collections.Sets; +import org.labkey.api.data.dialect.MockSqlDialect; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.dataiterator.SimpleTranslator; +import org.labkey.api.dataiterator.TableInsertUpdateDataIterator; +import org.labkey.api.exp.MvColumn; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.list.ListDefinition; +import org.labkey.api.exp.list.ListService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainKind; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.DomainUtil; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.gwt.client.model.GWTDomain; +import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; +import org.labkey.api.query.AliasManager; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.security.User; +import org.labkey.api.util.DateUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JunitUtil; +import org.labkey.api.util.TestContext; +import org.labkey.api.util.logging.LogHelper; + +import java.sql.Connection; +import java.sql.SQLException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Stream; + +import static java.util.Objects.requireNonNull; +import static org.labkey.api.util.JunitUtil.deleteTestContainer; + +public class StatementUtils +{ + private static final Logger _log = LogHelper.getLogger(StatementUtils.class, "SQL insert/update/delete generation"); + + public enum Operation {insert, update, merge} + + // configuration parameters + private Operation _operation; + private SqlDialect _dialect; + private final TableInfo _targetTable; + private Set _keyColumnNames = null; // override the primary key of _table + private Set _skipColumnNames = Set.of(); + private final Set _dontUpdateColumnNames = new CaseInsensitiveHashSet(); + private boolean _updateBuiltInColumns = false; // default to false, this should usually be handled by StandardDataIteratorBuilder + private boolean _selectIds = false; + private boolean _selectObjectUri = false; + private boolean _allowUpdateAutoIncrement = false; + private boolean _preferPKOverObjectUriAsKey = false; + + // variable/parameter tracking helpers + private boolean useVariables = false; + private final Map _constants = new CaseInsensitiveHashMap<>(); + final Map parameters = new CaseInsensitiveMapWrapper<>(new LinkedHashMap<>()); + + // ColumnTracker is used for test instrumentation and contains sets of column names that were included in + // given operations (insert, update, select) after running createStatement(). + // This is intended for test instrumentation use only. + private record ColumnTracker(Set insertColumns, Set updateColumns, Set selectColumns) + { + public ColumnTracker() + { + this(new CaseInsensitiveHashSet(), new CaseInsensitiveHashSet(), new CaseInsensitiveHashSet()); + } + } + + private ColumnTracker _columnTracker; + + // + // builder style methods + // + + //Vocabulary adhoc properties + private Set _vocabularyProperties = new HashSet<>(); + + public StatementUtils(@NotNull Operation op, @NotNull TableInfo table) + { + _operation = op; + _dialect = table.getSqlDialect(); + _targetTable = table; + } + + public StatementUtils dialect(SqlDialect dialect) + { + _dialect = dialect; + return this; + } + + public StatementUtils operation(@NotNull Operation op) + { + _operation = op; + return this; + } + + public StatementUtils constants(@NotNull Map constants) + { + _constants.putAll(constants); + return this; + } + + public StatementUtils keys(Set keyNames) + { + _keyColumnNames = keyNames; + return this; + } + + public StatementUtils skip(Set skip) + { + _skipColumnNames = null==skip ? Set.of() : skip; + return this; + } + + public StatementUtils noupdate(Set noupdate) + { + if (null != noupdate) + _dontUpdateColumnNames.addAll(noupdate); + return this; + } + + public StatementUtils updateBuiltinColumns(boolean b) + { + _updateBuiltInColumns = b; + return this; + } + + public StatementUtils selectIds(boolean b) + { + _selectIds = b; + return this; + } + + public StatementUtils selectObjectUri(boolean b) + { + _selectObjectUri = b; + return this; + } + + public StatementUtils allowSetAutoIncrement(boolean b) + { + _allowUpdateAutoIncrement = b; + return this; + } + + public StatementUtils setVocabularyProperties(Set vocabularyProperties) + { + _vocabularyProperties = vocabularyProperties; + return this; + } + + public StatementUtils setPreferPKOverObjectUriAsKey(boolean preferPKOverObjectUriAsKey) + { + _preferPKOverObjectUriAsKey = preferPKOverObjectUriAsKey; + return this; + } + + private static StatementUtils insertStatement(TableInfo table, boolean selectIds, boolean autoFillDefaultColumns) + { + return new StatementUtils(Operation.insert, table) + .updateBuiltinColumns(autoFillDefaultColumns) + .selectIds(selectIds); + } + + /** + * Create a reusable SQL Statement for inserting rows into a labkey relationship. The relationship + * persisted directly in the database (SchemaTableInfo), or via the OntologyManager tables. + *

+ * QueryService shouldn't really know about the internals of exp.Object and exp.ObjectProperty etc. + * However, I can only keep so many levels of abstraction in my head at once. + *

+ * NOTE: this is currently fairly expensive for updating one row into an Ontology stored relationship on Postgres. + * This shouldn't be a big problem since we don't usually need to optimize the one-row case, and we're moving + * to provisioned tables for major datatypes. + */ + public static ParameterMapStatement insertStatement(Connection conn, TableInfo table, @Nullable Container c, @Nullable User user, boolean selectIds, boolean autoFillDefaultColumns) throws SQLException + { + return insertStatement(table, selectIds, autoFillDefaultColumns) + .createStatement(conn, c, user); + } + + private static StatementUtils updateStatement(TableInfo table, boolean selectIds, boolean autoFillDefaultColumns) + { + return new StatementUtils(Operation.update, table) + .updateBuiltinColumns(autoFillDefaultColumns) + .selectIds(selectIds); + } + + /** + * Create a reusable SQL Statement for updating rows into a labkey relationship. The relationship + * persisted directly in the database (SchemaTableInfo), or via the OntologyManager tables. + *

+ * QueryService shouldn't really know about the internals of exp.Object and exp.ObjectProperty etc. + * However, I can only keep so many levels of abstraction in my head at once. + *

+ * NOTE: this is currently fairly expensive for updating one row into an Ontology stored relationship on Postgres. + * This shouldn't be a big problem since we don't usually need to optimize the one-row case, and we're moving + * to provisioned tables for major datatypes. + */ + public static ParameterMapStatement updateStatement(Connection conn, TableInfo table, @Nullable Container c, User user, boolean selectIds, boolean autoFillDefaultColumns) throws SQLException + { + return updateStatement(table, selectIds, autoFillDefaultColumns) + .createStatement(conn, c, user); + } + + private static StatementUtils mergeStatement(TableInfo table, @Nullable Set keyNames, @Nullable Set skipColumnNames, @Nullable Set dontUpdate, boolean selectIds, boolean autoFillDefaultColumns, boolean supportsAutoIncrementKey) + { + return new StatementUtils(Operation.merge, table) + .keys(keyNames) + .skip(skipColumnNames) + .allowSetAutoIncrement(supportsAutoIncrementKey) + .noupdate(dontUpdate) + .updateBuiltinColumns(autoFillDefaultColumns) + .selectIds(selectIds); + } + + public static ParameterMapStatement mergeStatement(Connection conn, TableInfo table, @Nullable Set keyNames, @Nullable Set skipColumnNames, @Nullable Set dontUpdate, @Nullable Container c, @Nullable User user, boolean selectIds, boolean autoFillDefaultColumns, boolean supportsAutoIncrementKey) throws SQLException + { + return mergeStatement(table, keyNames, skipColumnNames, dontUpdate, selectIds, autoFillDefaultColumns, supportsAutoIncrementKey) + .createStatement(conn, c, user); + } + + /* + * Parameter and Variable helpers + */ + + private static class ParameterHolder + { + ParameterHolder(Parameter p) + { + this.p = p; + _columnInfo = null; + } + + ParameterHolder(Parameter p, ColumnInfo c) + { + this.p = p; + _columnInfo = c; + } + + int getScale() + { + var type = requireNonNull(p.getType()); + if (null == _columnInfo || _columnInfo.getScale() <= 0) + return -1; + // GUID.isText()==true + if (JdbcType.GUID != type && (type.isText() || type.isDecimal())) + return _columnInfo.getScale(); + return -1; + } + + int getPrecision() + { + return null==_columnInfo ? -1 : _columnInfo.getPrecision(); + } + + final Parameter p; + final ColumnInfo _columnInfo; + String variableName = null; + Object constantValue = null; + boolean isConstant = false; + } + + private final static String pgRowVarPrefix = "$1."; + private String makeVariableName(String name) + { + String shortName = StringUtils.substring(name,0,32); // name is just for readability, make it short + String uniquePrefix = (_dialect.isSqlServer() ? "@" : pgRowVarPrefix) + ("p" + (parameters.size()+1) + "_"); + return uniquePrefix + AliasManager.makeLegalName(shortName, _dialect, true, uniquePrefix.length()); + } + + private String makePgRowTypeName(String variableName) + { + return StringUtils.substringAfter(variableName, pgRowVarPrefix); + } + + private ParameterHolder createParameter(ColumnInfo c) + { + ParameterHolder ph = parameters.get(c.getName()); + if (null == ph) + { + ph = new ParameterHolder(new Parameter(c.getName(), c.getPropertyURI(), null, c.getJdbcType()), c); + // NOTE: earlier DataIterator should probably split file into two columns: attachment_name, attachment_body + if (c.getInputType().equalsIgnoreCase("file") && c.getJdbcType() == JdbcType.VARCHAR) + ph.p.setFileAsName(true); + initParameterHolder(ph); + parameters.put(c.getName(), ph); + } + return ph; + } + + private void initParameterHolder(ParameterHolder ph) + { + String name = ph.p.getName(); + JdbcType type = ph.p.getType(); + assert null != type; + if (_constants.containsKey(name)) + { + Object value = Parameter.getValueToBind(_constants.get(name), type); + if (null == value || value instanceof Number || value instanceof String || value instanceof java.util.Date) + { + ph.isConstant = true; + ph.constantValue = value; + } + } + ph.variableName = makeVariableName(name); + } + + + private ParameterHolder createParameter(String name, JdbcType type) + { + ParameterHolder ph = parameters.get(name); + if (null == ph) + { + ph = new ParameterHolder(new Parameter(name, type)); + initParameterHolder(ph); + parameters.put(name, ph); + } + return ph; + } + + + private ParameterHolder createParameter(String name, String uri, JdbcType type) + { + ParameterHolder ph = parameters.get(name); + if (null == ph) + { + ph = new ParameterHolder(new Parameter(name, uri, null, type)); + initParameterHolder(ph); + parameters.put(name, ph); + } + return ph; + } + + private SQLFragment appendParameterOrVariable(SQLFragment f, ParameterHolder ph) + { + if (ph.isConstant) + { + toLiteral(f, ph.constantValue); + } + else if (useVariables) + { + f.append(ph.variableName); + } + else + { + f.append("?"); + f.add(ph.p); + } + return f; + } + + private SQLFragment appendPropertyValue(SQLFragment f, DomainProperty dp, ParameterHolder p) + { + if (dp.getJdbcType() == JdbcType.BOOLEAN) + { + f.append("CASE CAST("); + appendParameterOrVariable(f, p); + f.append(" AS ").append(_dialect.getBooleanDataType()).append(")") + .append(" WHEN ").append(_dialect.getBooleanTRUE()).append(" THEN 1.0 ") + .append(" WHEN ").append(_dialect.getBooleanFALSE()).append(" THEN 0.0 ") + .append(" ELSE NULL END"); + return f; + } + else + { + return appendParameterOrVariable(f, p); + } + } + + private void appendSQLFObjectProperty(SQLFragment sqlfObjectProperty, DomainProperty dp, String objectIdVar, String ifTHEN, String ifEND) + { + PropertyType propertyType = dp.getPropertyDescriptor().getPropertyType(); + ParameterHolder v = createParameter(dp.getName(), dp.getPropertyURI(), propertyType.getJdbcType()); + ParameterHolder mv = createParameter(dp.getName()+ MvColumn.MV_INDICATOR_SUFFIX, dp.getPropertyURI() + MvColumn.MV_INDICATOR_SUFFIX, JdbcType.VARCHAR); + sqlfObjectProperty.append("IF ("); + appendPropertyValue(sqlfObjectProperty, dp, v); + sqlfObjectProperty.append(" IS NOT NULL"); + if (dp.isMvEnabled()) + { + sqlfObjectProperty.append(" OR "); + appendParameterOrVariable(sqlfObjectProperty, mv); + sqlfObjectProperty.append(" IS NOT NULL"); + } + sqlfObjectProperty.append(")"); + sqlfObjectProperty.append(ifTHEN); + sqlfObjectProperty.append("INSERT INTO exp.ObjectProperty (objectid, propertyid, typetag, mvindicator, "); + sqlfObjectProperty.append(propertyType.getValueTypeColumn()); + sqlfObjectProperty.append(") VALUES ("); + sqlfObjectProperty.append(objectIdVar); + sqlfObjectProperty.append(",").appendValue(dp.getPropertyId()); + sqlfObjectProperty.append(",").appendStringLiteral(String.valueOf(propertyType.getStorageType()), _dialect); + sqlfObjectProperty.append(","); + if (dp.isMvEnabled()) + appendParameterOrVariable(sqlfObjectProperty, mv); + else + sqlfObjectProperty.append("NULL"); + sqlfObjectProperty.append(","); + appendPropertyValue(sqlfObjectProperty, dp, v); + sqlfObjectProperty.append(")").appendEOS(); + sqlfObjectProperty.append(ifEND); + sqlfObjectProperty.appendEOS(); + } + + private void appendSQLFDeleteObjectProperty(SQLFragment sqlfDelete, String objectIdVar, List domainProperties, Set vocabularyProperties) + { + var properties = null == domainProperties ? vocabularyProperties : domainProperties; + sqlfDelete.append("DELETE FROM exp.ObjectProperty WHERE ObjectId = "); + sqlfDelete.append(objectIdVar); + sqlfDelete.append(" AND PropertyId IN ("); + String separator = ""; + for (DomainProperty property : properties) + { + sqlfDelete.append(separator); + separator = ", "; + sqlfDelete.appendValue(property.getPropertyId()); + } + sqlfDelete.append(")").appendEOS(); + } + + private void setObjectUriPreselect(SQLFragment sqlfPreselectObject, TableInfo table, LinkedHashMap keys, String objectURIVar, String objectURIColumnName, ParameterHolder objecturiParameter) + { + String setKeyword = _dialect.isPostgreSQL() ? "" : "SET "; + if (Operation.merge == _operation || Operation.update == _operation) + { + // this seems overkill actually, but I'm focused on optimizing insert right now (MAB) + sqlfPreselectObject.append(setKeyword).append(objectURIVar).append(" = COALESCE(("); + sqlfPreselectObject.append("SELECT ").appendIdentifier(table.getColumn(objectURIColumnName).getSelectIdentifier()); + sqlfPreselectObject.append(" FROM ").append(table.getSQLName()); + sqlfPreselectObject.append(getPkWhereClause(keys)); + sqlfPreselectObject.append("),"); + appendParameterOrVariable(sqlfPreselectObject, objecturiParameter); + sqlfPreselectObject.append(")").appendEOS(); + + } + else + { + sqlfPreselectObject.append(setKeyword).append(objectURIVar).append(" = "); + appendParameterOrVariable(sqlfPreselectObject, objecturiParameter); + sqlfPreselectObject.appendEOS(); + } + } + + public ParameterMapStatement createStatement(Connection conn, @Nullable Container c, User user) throws SQLException + { + ParameterMapStatement statement = null; + try + { + statement = createStatement(conn, c, user, false); + } + catch (TableInsertUpdateDataIterator.NoUpdatableColumnInDataException e) + { + // ignore error + } + return statement; + } + + public ParameterMapStatement createStatement(Connection conn, @Nullable Container c, User user, boolean checkUpdatableColumns) throws SQLException, TableInsertUpdateDataIterator.NoUpdatableColumnInDataException + { + if (!(_targetTable instanceof UpdateableTableInfo updatable)) + throw new IllegalArgumentException("Table must be an UpdateableTableInfo"); + + TableInfo table = updatable.getSchemaTableInfo(); + + if (table.getTableType() != DatabaseTableType.TABLE) + throw new IllegalArgumentException("Table must be a database table"); + if (null == table.getMetaDataIdentifier()) + throw new IllegalArgumentException("Table must have a metadata identifier"); + + if (Operation.merge == _operation) + { + if (!_dialect.isPostgreSQL() && !_dialect.isSqlServer()) + throw new IllegalArgumentException("Merge is only supported/tested on postgres and sql server"); + } + + useVariables = Operation.merge == _operation; + String ifTHEN = _dialect.isSqlServer() ? " BEGIN " : " THEN "; + String ifEND = _dialect.isSqlServer() ? " END " : " END IF "; + + if (null != c) + { + assert null == _constants.get("container") || c.getId().equals(_constants.get("container")); + if (null == _constants.get("container")) + _constants.put("container", c.getId()); + } + + String objectURIColumnName = updatable.getObjectUriType() == UpdateableTableInfo.ObjectUriType.schemaColumn + ? updatable.getObjectURIColumnName() + : "objecturi"; + ParameterHolder objecturiParameter = null; + if (null != objectURIColumnName) + objecturiParameter = createParameter(objectURIColumnName, JdbcType.VARCHAR); + + // + // Keys for UPDATE or MERGE + // + LinkedHashMap keys = getKeys(updatable, table, objectURIColumnName, _keyColumnNames, _preferPKOverObjectUriAsKey); + + // + // exp.Objects INSERT + // + + SQLFragment sqlfDeclare = new SQLFragment(); + SQLFragment sqlfPreselectObject = new SQLFragment(); + SQLFragment sqlfInsertObject = new SQLFragment(); + SQLFragment sqlfSelectObject = new SQLFragment(); + SQLFragment sqlfObjectProperty = new SQLFragment(); + SQLFragment sqlfDelete = new SQLFragment(); + + Domain domain = updatable.getDomain(); + DomainKind domainKind = updatable.getDomainKind(); + List properties = Collections.emptyList(); + + boolean hasObjectURIColumn = objectURIColumnName != null && table.getColumn(objectURIColumnName) != null; + boolean alwaysInsertExpObject = (hasObjectURIColumn && updatable.isAlwaysInsertExpObject()) && Operation.update != _operation; + if (hasObjectURIColumn) + _dontUpdateColumnNames.add(objectURIColumnName); +// TODO Should we add created and createdby? Or make the caller decide? + if (Operation.update == _operation) + { + _dontUpdateColumnNames.add("Created"); + _dontUpdateColumnNames.add("CreatedBy"); + } + + String objectIdVar = null; + String objectURIVar = null; + boolean objectUriPreselectSet = false; + boolean isMaterializedDomain = null != domain && null != domainKind && StringUtils.isNotEmpty(domainKind.getStorageSchemaName()); + if (alwaysInsertExpObject || (null != domain && !isMaterializedDomain) || !_vocabularyProperties.isEmpty()) + { + properties = (null==domain||isMaterializedDomain) ? Collections.emptyList() : domain.getProperties(); + + if (alwaysInsertExpObject || !properties.isEmpty() || !_vocabularyProperties.isEmpty()) + { + if (!_dialect.isPostgreSQL() && !_dialect.isSqlServer()) + throw new IllegalStateException("Domains are only supported for sql server and postgres"); + + objectIdVar = _dialect.isPostgreSQL() ? "_$objectid$_" : "@_objectid_"; + sqlfDeclare.append("DECLARE ").append(objectIdVar).append(" BIGINT").appendEOS(); + objectURIVar = _dialect.isPostgreSQL() ? "_$objecturi$_" : "@_objecturi_"; + sqlfDeclare.append("DECLARE ").append(objectURIVar).append(" ").append(_dialect.getSqlTypeName(JdbcType.VARCHAR)).append("(300)").appendEOS(); + useVariables |= _dialect.isPostgreSQL(); + + ParameterHolder containerParameter = createParameter("container", JdbcType.GUID); + + // Insert a new row in exp.Object if there isn't already a row for this object + + // Grab the object's ObjectId based on the pk of the base table + if (hasObjectURIColumn || !_vocabularyProperties.isEmpty()) + { + setObjectUriPreselect(sqlfPreselectObject, table, keys, objectURIVar, objectURIColumnName, objecturiParameter); + objectUriPreselectSet = true; + } + + SQLFragment sqlfWhereObjectURI = new SQLFragment(); + sqlfWhereObjectURI.append("(ObjectURI = ").append(objectURIVar).append(")"); + + // In the update case, it's still possible that there isn't a row in exp.Object - there might have been + // no properties in the domain when the row was originally inserted + sqlfInsertObject.append("INSERT INTO exp.Object (container, objecturi, ownerobjectid) "); + sqlfInsertObject.append("SELECT "); + appendParameterOrVariable(sqlfInsertObject, containerParameter); + sqlfInsertObject.append(" AS Container,"); + appendParameterOrVariable(sqlfInsertObject, objecturiParameter); + sqlfInsertObject.append(" AS ObjectURI, "); + Long ownerObjectId = updatable.getOwnerObjectId(); + sqlfInsertObject.append( null == ownerObjectId ? "NULL" : String.valueOf(ownerObjectId) ).append(" AS OwnerObjectId"); + sqlfInsertObject.append(" WHERE NOT EXISTS (SELECT ObjectURI FROM exp.Object WHERE Container = "); + appendParameterOrVariable(sqlfInsertObject, containerParameter); + sqlfInsertObject.append(" AND ").append(sqlfWhereObjectURI).append(")").appendEOS(); + + // re-grab the object's ObjectId, in case it was just inserted + sqlfSelectObject.append(_dialect.isPostgreSQL() ? "" : "SET ").append(objectIdVar).append(" = ("); + sqlfSelectObject.append("SELECT ObjectId FROM exp.Object WHERE Container = "); + appendParameterOrVariable(sqlfSelectObject, containerParameter); + sqlfSelectObject.append(" AND ").append(sqlfWhereObjectURI).append(")").appendEOS(); + + if (Operation.insert != _operation && (!properties.isEmpty() || !_vocabularyProperties.isEmpty())) + { + // Clear out any existing property values for this domain + if (!properties.isEmpty()) + { + appendSQLFDeleteObjectProperty(sqlfDelete, objectIdVar, properties, null); + } + + // Clear out any existing ad hoc property + if (!_vocabularyProperties.isEmpty()) + { + appendSQLFDeleteObjectProperty(sqlfDelete, objectIdVar, null, _vocabularyProperties); + } + } + } + } + + if (_selectObjectUri) + { + if (objectURIVar == null) + { + objectURIVar = _dialect.isPostgreSQL() ? "_$objecturi$_" : "@_objecturi_"; + sqlfDeclare.append("DECLARE ").append(objectURIVar).append(" ").append(_dialect.getSqlTypeName(JdbcType.VARCHAR)).append("(300)").appendEOS(); + } + + if (!objectUriPreselectSet && (hasObjectURIColumn || !_vocabularyProperties.isEmpty())) + { + setObjectUriPreselect(sqlfPreselectObject, table, keys, objectURIVar, objectURIColumnName, objecturiParameter); + } + } + + + // + // BASE TABLE INSERT() + // + + ColumnInfo col; + List cols = new ArrayList<>(); + List values = new ArrayList<>(); + Set done = Sets.newCaseInsensitiveHashSet(); + + if (_updateBuiltInColumns && Operation.update != _operation) + { + col = table.getColumn("Owner"); + if (null != col && null != user) + { + cols.add(col); + values.add(new SQLFragment().appendValue(user.getUserId())); + done.add("Owner"); + } + col = table.getColumn("CreatedBy"); + if (null != col && null != user) + { + cols.add(col); + values.add(new SQLFragment().appendValue(user.getUserId())); + done.add("CreatedBy"); + } + col = table.getColumn("Created"); + if (null != col) + { + cols.add(col); + values.add(new SQLFragment().appendValue(new SQLFragment.NowTimestamp())); + done.add("Created"); + } + } + + ColumnInfo colModifiedBy = table.getColumn("ModifiedBy"); + if (_updateBuiltInColumns && null != colModifiedBy && null != user) + { + cols.add(colModifiedBy); + values.add(new SQLFragment().appendValue(user.getUserId())); + done.add("ModifiedBy"); + } + + ColumnInfo colModified = table.getColumn("Modified"); + if (_updateBuiltInColumns && null != colModified) + { + cols.add(colModified); + values.add(new SQLFragment().appendValue(new SQLFragment.NowTimestamp())); + done.add("Modified"); + } + ColumnInfo colVersion = table.getVersionColumn(); + if (_updateBuiltInColumns && null != colVersion && !done.contains(colVersion.getName())) + { + SQLFragment expr = colVersion.getVersionUpdateExpression(); + if (null != expr) + { + cols.add(colVersion); + values.add(expr); + done.add(colVersion.getName()); + } + } + + String objectIdColumnName = StringUtils.trimToNull(updatable.getObjectIdColumnName()); + ColumnInfo autoIncrementColumn = null; + CaseInsensitiveHashMap remap = updatable.remapSchemaColumns(); + if (null == remap) + remap = CaseInsensitiveHashMap.of(); + + for (ColumnInfo column : table.getColumns()) + { + if (column instanceof WrappedColumn || column.isCalculated()) + continue; + // if we're allowing the caller to set the auto-increment column, then treat like a regular column + if (column.isAutoIncrement() && !_allowUpdateAutoIncrement) + { + autoIncrementColumn = column; + continue; + } + if (column.isVersionColumn() && column != colModified) + continue; + String name = column.getName(); + if (done.contains(name)) + continue; + done.add(name); + ColumnInfo updatableColumn = updatable.getColumn(column.getName()); + if (updatableColumn != null && updatableColumn.hasDbSequence()) + _dontUpdateColumnNames.add(column.getName()); + + SQLFragment valueSQL = new SQLFragment(); + if (column.getName().equalsIgnoreCase(objectIdColumnName)) + { + valueSQL.append(objectIdVar); + } + else if (column.getName().equalsIgnoreCase(updatable.getObjectURIColumnName()) && null != objecturiParameter) + { + appendParameterOrVariable(valueSQL, objecturiParameter); + } + else + { + if (null != _skipColumnNames && _skipColumnNames.contains(Objects.toString(remap.get(name),name))) + continue; + ParameterHolder ph = createParameter(column); + appendParameterOrVariable(valueSQL, ph); + } + cols.add(column); + values.add(valueSQL); + } + + boolean selectAutoIncrement = false; + + assert cols.size() == values.size() : cols.size() + " columns and " + values.size() + " values - should match"; + + // + // INSERT + // + + String comma; + String rowIdVar = null; + SQLFragment sqlfInsertInto = new SQLFragment(); + + // Construct a new column tracker for test instrumentation + _columnTracker = new ColumnTracker(); + + if (Operation.insert == _operation || Operation.merge == _operation) + { + // Create a standard INSERT INTO table (col1, col2) VALUES (val1, val2) statement + // or (for degenerate, empty values case) INSERT INTO table VALUES (DEFAULT) + sqlfInsertInto.append("INSERT INTO ").append(table.getSQLName()); + + if (values.isEmpty()) + { + sqlfInsertInto.append("\nVALUES (DEFAULT)"); + } + else + { + sqlfInsertInto.append(" ("); + comma = ""; + for (ColumnInfo colInfo : cols) + { + sqlfInsertInto.append(comma); + comma = ", "; + sqlfInsertInto.appendIdentifier(colInfo.getSelectIdentifier()); + _columnTracker.insertColumns.add(colInfo.getName()); + } + sqlfInsertInto.append(")"); + + sqlfInsertInto.append("\nSELECT "); + comma = ""; + for (SQLFragment valueSQL : values) + { + sqlfInsertInto.append(comma); + comma = ", "; + sqlfInsertInto.append(valueSQL); + } + } + + if (_selectIds && null != autoIncrementColumn) + { + selectAutoIncrement = true; + if (useVariables) + rowIdVar = "_rowid_"; + rowIdVar = _dialect.addReselect(sqlfInsertInto, autoIncrementColumn, rowIdVar); + if (useVariables) + sqlfDeclare.append("DECLARE ").append(rowIdVar).append(" BIGINT").appendEOS(); // CONSIDER: Move this into addReselect()? + } + + if (_selectObjectUri && hasObjectURIColumn) + { + _dialect.addReselect(sqlfInsertInto, table.getColumn(objectURIColumnName), objectURIVar); + } + } + + // + // UPDATE + // + + SQLFragment sqlfUpdate = new SQLFragment(); + if (Operation.update == _operation || Operation.merge == _operation) + { + // Create a standard UPDATE table SET col1 = val1, col2 = val2 statement + sqlfUpdate.append("UPDATE ").append(table.getSQLName()).append("\nSET "); + comma = ""; + int updateCount = 0; + for (int i = 0; i < cols.size(); i++) + { + col = cols.get(i); + FieldKey fk = col.getFieldKey(); + if (keys.containsKey(fk)) + continue; + + // Issue 52666: Check column remapping when looking for columns to not update + String colName = col.getName(); + if (_dontUpdateColumnNames.contains(colName) || (remap.containsKey(colName) && _dontUpdateColumnNames.contains(remap.get(colName)))) + continue; + + sqlfUpdate.append(comma); + comma = ", "; + sqlfUpdate.appendIdentifier(col.getSelectIdentifier()); + sqlfUpdate.append(" = "); + sqlfUpdate.append(values.get(i)); + _columnTracker.updateColumns.add(col.getName()); + updateCount++; + } + + if (Operation.update == _operation && updateCount == 0) + { + if (checkUpdatableColumns) + throw new TableInsertUpdateDataIterator.NoUpdatableColumnInDataException(table.getName()); + + sqlfUpdate.appendIdentifier(keys.values().iterator().next().getSelectIdentifier()); + sqlfUpdate.append(" = 'noop' WHERE 1 <> 1").appendEOS(); + } + else + { + sqlfUpdate.append(getPkWhereClause(keys)); + sqlfUpdate.appendEOS(); + } + + if (Operation.merge == _operation) + { + // updateCount can equal 0. This happens particularly when inserting into junction tables where + // there are two columns and both are in the primary key + if (0 == updateCount) + { + sqlfUpdate = new SQLFragment(); + sqlfInsertInto.append("\nWHERE NOT EXISTS (SELECT * FROM ").append(table.getSQLName()); + sqlfInsertInto.append(getPkWhereClause(keys)); + sqlfInsertInto.append(")"); + } + else + { + sqlfUpdate.append("IF "); + sqlfUpdate.append(_dialect.isSqlServer() ? "@@ROWCOUNT=0" : "NOT FOUND"); + sqlfUpdate.append(ifTHEN).append("\n\t"); + + sqlfInsertInto.appendEOS(); + sqlfInsertInto.append(ifEND); + } + } + } + + if (Operation.insert == _operation || Operation.merge == _operation) + sqlfInsertInto.appendEOS(); + + SQLFragment sqlfSelectIds = null; + + if ((_selectIds && (null != objectIdVar || null != rowIdVar)) || (_selectObjectUri && null != objectURIVar)) + { + sqlfSelectIds = new SQLFragment("SELECT "); + comma = ""; + if (_selectIds) + { + if (null != rowIdVar) + { + sqlfSelectIds.append(rowIdVar); + _columnTracker.selectColumns.add(rowIdVar); + comma = ","; + } + if (null != objectIdVar) + { + sqlfSelectIds.append(comma).append(objectIdVar); + _columnTracker.selectColumns.add(objectIdVar); + comma = ","; + } + } + + if (_selectObjectUri && null != objectURIVar) + { + sqlfSelectIds.append(comma).append(objectURIVar); + _columnTracker.selectColumns.add(objectIdVar); + } + } + + // + // ObjectProperty + // + + if (!properties.isEmpty()) + { + Set skip = updatable.skipProperties(); + if (null != skip) + done.addAll(skip); + + for (DomainProperty dp : properties) + { + // ignore property that 'wraps' a hard column + if (done.contains(dp.getName())) + continue; + appendSQLFObjectProperty(sqlfObjectProperty, dp, objectIdVar, ifTHEN, ifEND); + } + } + + if (!_vocabularyProperties.isEmpty()) + { + for (DomainProperty vocProp: _vocabularyProperties) + { + appendSQLFObjectProperty(sqlfObjectProperty, vocProp, objectIdVar, ifTHEN, ifEND); + } + } + + // + // PREPARE + // + + ParameterMapStatement ret; + + if (!useVariables) + { + SQLFragment script = new SQLFragment(); + Stream.of(sqlfDeclare, sqlfPreselectObject, sqlfInsertObject, sqlfSelectObject, sqlfDelete, sqlfUpdate, sqlfInsertInto, sqlfObjectProperty, sqlfSelectIds) + .filter(f -> null != f && !f.isEmpty()) + .forEach(script::append); + ret = new ParameterMapStatement(table.getSchema().getScope(), conn, script, remap); + } + else if (_dialect.isSqlServer()) + { + if (!parameters.isEmpty()) + { + SQLFragment select = new SQLFragment(); + sqlfDeclare.append("DECLARE "); + select.append("SELECT "); + comma = ""; + for (Map.Entry e : parameters.entrySet()) + { + ParameterHolder ph = e.getValue(); + sqlfDeclare.append(comma); + String variable = sqlServerVariableDeclaration(sqlfDeclare, ph); + select.append(comma).append(variable).append("=?"); + select.add(ph.p); + comma = ", "; + } + sqlfDeclare.appendEOS(); + sqlfDeclare.append(select); + sqlfDeclare.appendEOS(); + } + SQLFragment script = new SQLFragment(); + Stream.of(sqlfDeclare, sqlfPreselectObject, sqlfInsertObject, sqlfSelectObject, sqlfDelete, sqlfUpdate, sqlfInsertInto, sqlfObjectProperty, sqlfSelectIds) + .filter(f -> null != f && !f.isEmpty()) + .forEach(script::append); + _log.debug(script.toDebugString()); + ret = new ParameterMapStatement(table.getSchema().getScope(), conn, script, remap); + } + else + { + // wrap in a function with a single ROW() constructor argument + SQLFragment fn = new SQLFragment(); + String fnName = _dialect.getGlobalTempTablePrefix() + "fn_" + GUID.makeHash(); + TempTableTracker.track(fnName, fn); + + String typeName = fnName + "type"; + fn.append("CREATE TYPE ").append(typeName).append(" AS ("); + // TODO d.execute() doesn't handle temp schema + SQLFragment call = new SQLFragment(); + call.append(fnName).append("(ROW("); + comma = ""; + for (Map.Entry e : parameters.entrySet()) + { + ParameterHolder ph = e.getValue(); + String type = _dialect.getSqlTypeName(ph.p.getType()); + fn.append("\n").append(comma); + fn.append(makePgRowTypeName(ph.variableName)); + fn.append(" "); + fn.append(type); + // For PG (29687) we need the length for CHAR type + if (_dialect.isPostgreSQL() && JdbcType.CHAR.equals(ph.p.getType())) + fn.append("(").appendValue(ph.getScale()).append(")"); + call.append(comma).append("?"); + call.add(ph.p); + comma = ","; + } + fn.append("\n)").appendEOS(); + fn.append("CREATE FUNCTION ").append(fnName).append("(").append(typeName).append(") "); + fn.append("RETURNS "); + if (null != sqlfSelectIds) + fn.append("SETOF RECORD"); + else + fn.append("void"); + String quoteToken = "$x" + GUID.makeHash() + "$"; + fn.append(" AS ").append(quoteToken).append("\n"); + call.append("))"); + + if (null != sqlfSelectIds) + { + call.insert(0, "SELECT * FROM "); + call.append(" AS x("); + String sep = ""; + + if (_selectIds) + { + if (null != rowIdVar) + { + call.append("A BIGINT"); + sep = ", "; + } + if (null != objectIdVar) + { + call.append(sep); + call.append("B BIGINT"); + sep = ", "; + } + } + + if (_selectObjectUri && null != objectURIVar) + { + call.append(sep); + call.append("C VARCHAR"); + } + + call.append(")").appendEOS(); + } + else + { + call.insert(0, "{call "); + call.append("}"); + } + + fn.append(sqlfDeclare); + + fn.append("BEGIN\n"); + fn.append("-- ").append(_operation.name()).append("\n"); + Stream.of(sqlfPreselectObject, sqlfInsertObject, sqlfSelectObject, sqlfDelete, sqlfUpdate, sqlfInsertInto, sqlfObjectProperty) + .filter(f -> null != f && !f.isEmpty()) + .forEach(fn::append); + if (null == sqlfSelectIds) + { + fn.append("RETURN").appendEOS(); + } + else + { + sqlfSelectIds.insert(0, "RETURN QUERY\n"); + fn.append(sqlfSelectIds); + fn.appendEOS(); + } + fn.append("END").appendEOS().append(" ").append(quoteToken).append(" LANGUAGE plpgsql").appendEOS(); + _log.debug(fn.toDebugString()); + _log.debug(call.toDebugString()); + final SQLFragment drop = new SQLFragment("DROP TYPE IF EXISTS ").append(typeName).append(" CASCADE").appendEOS(); + _log.debug(drop.toDebugString()); + new SqlExecutor(table.getSchema()).execute(fn); + ret = new ParameterMapStatement(table.getSchema().getScope(), conn, call, updatable.remapSchemaColumns()); + ret.setDebugSql(fn.getSQL() + "--\n" + call.toDebugString()); + ret.onClose(() -> { + try + { + new SqlExecutor(ExperimentService.get().getSchema()).execute(drop); + } + catch (Exception x) + { + _log.error("Error dropping custom rowtype for temp function.", x); + } + }); + } + + int selectIndex = 1; + + if (_selectIds) + { + // Why is one of these boolean and the other an index?? I don't know + ret.setSelectRowId(selectAutoIncrement); + + if (selectAutoIncrement) + selectIndex++; + + if (null != objectIdVar) + ret.setObjectIdIndex(selectIndex++); + } + + if (_selectObjectUri && null != objectURIVar) + ret.setObjectUriIndex(selectIndex); + + return ret; + } + + private static LinkedHashMap getKeys( + UpdateableTableInfo updatable, + TableInfo table, + String objectURIColumnName, + Set keyColumnNames, + boolean preferPKOverObjectUriAsKey + ) + { + LinkedHashMap keys = new LinkedHashMap<>(); + ColumnInfo col = table.getColumn("Container"); + + if (null != col) + keys.put(col.getFieldKey(), col); + + if (null != keyColumnNames && !keyColumnNames.isEmpty()) + { + for (String name : keyColumnNames) + { + col = table.getColumn(name); + if (null == col) + throw new IllegalArgumentException("Column not found: " + name); + keys.put(col.getFieldKey(), col); + } + } + else + { + // using objectURIColumnName preferentially to be backward compatible with OntologyManager.saveTabDelimited + // which in turn is only called by LuminexDataHandler.saveDataRows() + col = objectURIColumnName == null ? null : table.getColumn(objectURIColumnName); + if (null != col && !preferPKOverObjectUriAsKey) + keys.put(col.getFieldKey(), col); + else + { + // See Issue 26661 and Issue 41053 + // NOTE: IMO we should not be using updatable.getPkColumnNames() here! If the caller doesn't want to use the + // 'real' PK from the SchemaTableInfo for update/merge, then the alternate keys should be explicitly specified + // using StatementUtils.keys() + for (String pkName : updatable.getPkColumnNames()) + { + col = table.getColumn(pkName); + if (null == col) + throw new IllegalStateException("pk column not found: " + pkName); + keys.put(col.getFieldKey(), col); + } + } + } + + return keys; + } + + private SQLFragment getPkWhereClause(LinkedHashMap keys) + { + SQLFragment sqlfWherePK = new SQLFragment(); + sqlfWherePK.append("\nWHERE "); + String and = ""; + for (Map.Entry e : keys.entrySet()) + { + ColumnInfo keyCol = e.getValue(); + ParameterHolder keyColPh = createParameter(keyCol); + + sqlfWherePK.append(and); + sqlfWherePK.append("("); + sqlfWherePK.appendIdentifier(keyCol.getSelectIdentifier()); + sqlfWherePK.append(" = "); + appendParameterOrVariable(sqlfWherePK, keyColPh); + if (keyCol.isNullable()) + { + sqlfWherePK.append(" OR "); + sqlfWherePK.appendIdentifier(keyCol.getSelectIdentifier()); + sqlfWherePK.append(" IS NULL AND "); + appendParameterOrVariable(sqlfWherePK, keyColPh); + sqlfWherePK.append(" IS NULL"); + } + sqlfWherePK.append(")"); + and = " AND "; + } + return sqlfWherePK; + } + + private String sqlServerVariableDeclaration(SQLFragment sqlfDeclare, ParameterHolder ph) + { + assert(_dialect.isSqlServer()); + String variable = ph.variableName; + sqlfDeclare.append(variable); + sqlfDeclare.append(" "); + JdbcType jdbcType = ph.p.getType(); + assert null != jdbcType; + String type = _dialect.getSqlTypeName(jdbcType); + assert null != type; + + // Workaround - SQLServer doesn't support TEXT, NTEXT, or IMAGE as local variables in statements, but is OK with NVARCHAR(MAX) + if (jdbcType.isText()) + { + if ("NTEXT".equalsIgnoreCase(type) || "TEXT".equalsIgnoreCase(type) || ph.getScale()>4000) + type = "NVARCHAR(MAX)"; + else + type = "NVARCHAR(4000)"; + } + // Add scale and precision for decimal values specifying scale + else if (jdbcType.isDecimal() && ph.getScale() > 0) + { + type = type + "(" + ph.getPrecision() + "," + ph.getScale() + ")"; + } + + sqlfDeclare.append(type); + return variable; + } + + /* + * We could use SQLFragment.appendValue() for most of these. However, here it is important to force + * the use of inline literal values. SQLFragment.appendValue() does not guarantee that. + */ + private void toLiteral(SQLFragment f, Object value) + { + if (null == value) + { + f.append("NULL"); + return; + } + if (value instanceof Number) + { + f.append(value.toString()); + return; + } + if (value instanceof SQLFragment.NowTimestamp now) + { + f.appendValue(now); + return; + } + if (value instanceof java.sql.Date sqlDate) + { + f.append("{d ").append(_dialect.getStringHandler().quoteStringLiteral(DateUtil.formatIsoDate(sqlDate))).append("}"); + return; + } + else if (value instanceof java.util.Date date) + { + f.append("{ts ").append(_dialect.getStringHandler().quoteStringLiteral(DateUtil.formatIsoDateShortTime(date))).append("}"); + return; + } + assert value instanceof String; + f.append(_dialect.getStringHandler().quoteStringLiteral(String.valueOf(value))); + } + + @SuppressWarnings("JUnitMalformedDeclaration") + public static class TestCase extends Assert + { + final static String DATA_CLASS_NAME = "StatementUtilsTestDataClass"; + final static String VOCAB_DOMAIN_KIND = "Vocabulary"; // VocabularyDomainKind.KIND_NAME + final static String VOCAB_DOMAIN_NAME = "StatementUtilsVocabularyDomain"; + + final Container container; + final TableInfo dataClassTable; + final TableInfo principalsTable; + final UpdateableTableInfo testTable; + final User user; + final Set vocabParameters = CaseInsensitiveHashSet.of("Age", "AgeMVIndicator", "Color", "ColorMVIndicator"); + final Set vocabProps; + + // Flag to run tests against one or both (Postgres, SQL Server) SqlDialects. Set to false by default + // since tests are run in both environments in CI. See "otherSqlDialect". + final boolean runOtherDialect = false; + + // This is a mock SqlDialect that mocks the alternative SqlDialect configuration to the current configuration. + // So, if the tests are running in a Postgres environment, then this represents a SQL Server SqlDialect + // and vice versa. This is useful for getting code coverage across code paths for both dialects in a single + // test run. Enabled via the "runOtherDialect" flag. + final SqlDialect otherSqlDialect; + + public TestCase() throws Exception + { + container = JunitUtil.getTestContainer(); + user = TestContext.get().getUser(); + + dataClassTable = QueryService.get().getUserSchema(user, container, ExpSchema.SCHEMA_EXP_DATA).getTableOrThrow(DATA_CLASS_NAME); + principalsTable = DbSchema.get("core", DbSchemaType.Module).getTable("principals"); + testTable = DbSchema.get("test", DbSchemaType.Module).getTable("testtable2"); + + // Initialize vocab domain properties + { + var vocabDomainKind = PropertyService.get().getDomainKindByName(VOCAB_DOMAIN_KIND); + var vocabDomainURI = vocabDomainKind.generateDomainURI(null, VOCAB_DOMAIN_NAME, container, user); + var vocabDomain = PropertyService.get().getDomain(container, vocabDomainURI); + assertNotNull(vocabDomain); + vocabProps = Set.of(vocabDomain.getPropertyByName("Age"), vocabDomain.getPropertyByName("Color")); + } + + if (runOtherDialect) + { + SqlDialect defaultDialect = principalsTable.getSqlDialect(); + boolean isPostgres = defaultDialect.isPostgreSQL(); + + otherSqlDialect = new MockSqlDialect() + { + @Override + public String addReselect(SQLFragment sql, ColumnInfo column, @Nullable String proposedVariable) + { + return defaultDialect.addReselect(sql, column, proposedVariable); + } + + @Override + public String getGuidType() + { + return defaultDialect.getGuidType(); + } + + @Override + public @Nullable String getSqlTypeName(JdbcType type) + { + return defaultDialect.getSqlTypeName(type); + } + + @Override + public boolean isPostgreSQL() + { + // Returns true in SQL Server configured environments + return !isPostgres; + } + + @Override + public boolean isSqlServer() + { + // Returns true in Postgres configured environments + return isPostgres; + } + }; + } + else + { + otherSqlDialect = null; + } + } + + @BeforeClass + public static void createDomains() throws Exception + { + var container = JunitUtil.getTestContainer(); + var user = TestContext.get().getUser(); + + // Create a data class domain + ExperimentService.get().createDataClass(container, user, DATA_CLASS_NAME, null, List.of(new GWTPropertyDescriptor("aa", "int")), List.of(), null, null); + + // Create a vocabulary domain + { + GWTPropertyDescriptor prop1 = new GWTPropertyDescriptor(); + prop1.setRangeURI("int"); + prop1.setName("Age"); + prop1.setMvEnabled(true); + + GWTPropertyDescriptor prop2 = new GWTPropertyDescriptor(); + prop2.setRangeURI("string"); + prop2.setName("Color"); + + GWTDomain domain = new GWTDomain<>(); + domain.setName(VOCAB_DOMAIN_NAME); + domain.setFields(List.of(prop1, prop2)); + + DomainUtil.createDomain(VOCAB_DOMAIN_KIND, domain, null, container, user, VOCAB_DOMAIN_NAME, null, false); + } + } + + @AfterClass + public static void cleanup() + { + deleteTestContainer(); + } + + @Test + public void testToLiteral() + { + boolean isPostgres = principalsTable.getSqlDialect().isPostgreSQL(); + + var statement = new StatementUtils(Operation.insert, principalsTable); + Function runToLiteral = (value) -> { + var sql = new SQLFragment(); + statement.toLiteral(sql, value); + return sql; + }; + + var dateLong = 1749759500016L; // Thu Jun 12 2025 13:18:20 GMT-0700 (Pacific Daylight Time) + + // null value + var actual = runToLiteral.apply(null); + assertEquals(new SQLFragment("NULL"), actual); + + // Number + actual = runToLiteral.apply(1234567890); + assertEquals(new SQLFragment("1234567890"), actual); + + // NowTimestamp + var now = new SQLFragment.NowTimestamp(dateLong); + actual = runToLiteral.apply(now); + assertEquals(new SQLFragment().appendValue(now), actual); + + // sql.Date + var sqlDate = new java.sql.Date(dateLong); + var dateFormat = new SimpleDateFormat(DateUtil.getStandardDateFormatString()); + var expected = String.format(isPostgres ? "{d '%s'}" : "{d N'%s'}", dateFormat.format(sqlDate)); + + actual = runToLiteral.apply(sqlDate); + assertEquals(new SQLFragment(expected), actual); + + // util.Date + var utilDate = new java.util.Date(dateLong); + dateFormat = new SimpleDateFormat(DateUtil.getStandardDateTimeFormatString()); + expected = String.format(isPostgres ? "{ts '%s'}" : "{ts N'%s'}", dateFormat.format(utilDate)); + + actual = runToLiteral.apply(utilDate); + assertEquals(new SQLFragment(expected), actual); + } + + @Test + public void testCreateStatementValidation() throws Exception + { + try (var conn = getConnection()) + { + var nonUpdateTable = new VirtualTable<>(DbSchema.get("test", DbSchemaType.Module), "virtualInsanity", null); + + var exception = Assert.assertThrows(IllegalArgumentException.class, () -> new StatementUtils(Operation.merge, nonUpdateTable).createStatement(conn, container, user)); + assertEquals("Table must be an UpdateableTableInfo", exception.getMessage()); + + // Unreachable with current mocks +// var nonDatabaseTable = QueryService.get().getUserSchema(user, container, "core").getTableOrThrow("Principals"); +// exception = Assert.assertThrows(IllegalArgumentException.class, () -> new StatementUtils(Operation.merge, nonDatabaseTable).createStatement(conn, container, user)); +// assertEquals("Table must be a database table", exception.getMessage()); + +// exception = Assert.assertThrows(IllegalArgumentException.class, () -> { +// var noIdentifierTable = principalsTable.getMetaDataIdentifier(). +// new StatementUtils(Operation.merge, nonDatabaseTable).dialect(new MockSqlDialect()).createStatement(conn, container, user); +// }); +// assertEquals("Table must have a metadata identifier", exception.getMessage()); + + exception = Assert.assertThrows(IllegalArgumentException.class, () -> new StatementUtils(Operation.merge, principalsTable).dialect(new MockSqlDialect()).createStatement(conn, container, user)); + assertEquals("Merge is only supported/tested on postgres and sql server", exception.getMessage()); + } + } + + @Test + public void testGetKeys() + { + var containerFieldKey = FieldKey.fromParts("Container"); + var rowIdFieldKey = FieldKey.fromParts("RowId"); + var textFieldKey = FieldKey.fromParts("Text"); + + var updateTable = testTable; + var table = updateTable.getSchemaTableInfo(); + + // Pre-conditions + var pkColumnNames = new CaseInsensitiveHashSet(testTable.getPkColumnNames()); + assertEquals(2, pkColumnNames.size()); + assertTrue(pkColumnNames.contains(containerFieldKey.getName())); + assertTrue(pkColumnNames.contains(textFieldKey.getName())); + assertNotNull(testTable.getColumn(rowIdFieldKey)); + + // The "Container" column is always resolved if present on the table + var keys = StatementUtils.getKeys(updateTable, table, null, Set.of(containerFieldKey.getName()), false); + assertEquals(1, keys.size()); + assertTrue(keys.containsKey(containerFieldKey)); + + // The "Container" column is only resolved even when in the explicit name map + keys = StatementUtils.getKeys(updateTable, table, null, Set.of(containerFieldKey.getName()), false); + assertEquals(1, keys.size()); + assertTrue(keys.containsKey(containerFieldKey)); + + // The "Container" column is also resolved even when not in the explicit key column map. Other key columns are included as well. + keys = StatementUtils.getKeys(updateTable, table, null, Set.of(textFieldKey.getName()), false); + assertEquals(2, keys.size()); + assertTrue(keys.containsKey(containerFieldKey)); + assertTrue(keys.containsKey(textFieldKey)); + + // All explicitly named columns should resolve as columns on the table + var exception = Assert.assertThrows(IllegalArgumentException.class, () -> StatementUtils.getKeys(updateTable, table, null, Set.of(textFieldKey.getName(), "Beep"), false)); + assertEquals("Column not found: Beep", exception.getMessage()); + + // Furnish an explicit "objectURIColumnName" and expect it to be included when preferPKOverObjectUriAsKey = false + keys = StatementUtils.getKeys(updateTable, table, "RowId", null, false); + assertEquals(2, keys.size()); + assertTrue(keys.containsKey(containerFieldKey)); + assertTrue(keys.containsKey(rowIdFieldKey)); + + // Furnish an explicit "objectURIColumnName" and expect it to NOT be included when preferPKOverObjectUriAsKey = true + keys = StatementUtils.getKeys(updateTable, table, "RowId", null, true); + assertEquals(2, keys.size()); + assertTrue(keys.containsKey(containerFieldKey)); + assertTrue(keys.containsKey(textFieldKey)); + + keys = StatementUtils.getKeys(updateTable, table, null, null, false); + assertEquals(2, keys.size()); + assertTrue(keys.containsKey(containerFieldKey)); + assertTrue(keys.containsKey(textFieldKey)); + } + + @Test + public void testInsert() throws Exception + { + ParameterMapStatement m = null; + try (Connection conn = getConnection()) + { + m = insertStatement(conn, principalsTable, container, user, true, true); + m.close(); m = null; + + m = insertStatement(conn, testTable, container, user, true, true); + m.close(); m = null; + } + finally + { + if (null != m) + m.close(); + } + } + + @Test + public void testInsertWithExtensibleDomain() throws Exception + { + ParameterMapStatement m = null; + try (Connection conn = getConnection(dataClassTable)) + { + StatementUtils statement; + + // Insert + { + var validateInsert = new Function() + { + @Override + public Object apply(StatementUtils s) + { + boolean isPostgres = s._dialect.isPostgreSQL(); + + assertTrue(s._columnTracker.insertColumns.contains("Created")); + assertTrue(s._columnTracker.insertColumns.contains("CreatedBy")); + assertTrue(s._columnTracker.insertColumns.contains("Modified")); + assertTrue(s._columnTracker.insertColumns.contains("ModifiedBy")); + assertTrue(s._columnTracker.updateColumns.isEmpty()); + + if (isPostgres) + { + assertTrue(s._columnTracker.selectColumns.contains("_rowid_")); + assertTrue(s._columnTracker.selectColumns.contains("_$objectid$_")); + } + else + { + // Variables are not used in SQL Server + assertFalse(s._columnTracker.selectColumns.contains("_rowid_")); + assertTrue(s._columnTracker.selectColumns.contains("@_objectid_")); + } + + var parameterKeys = s.parameters.keySet(); + assertTrue(vocabParameters.stream().noneMatch(parameterKeys::contains)); + return null; + } + }; + + statement = insertStatement(dataClassTable, true, true); + m = statement.createStatement(conn, container, user); + m.close(); + m = null; + validateInsert.apply(statement); + + if (runOtherDialect) + { + statement = insertStatement(dataClassTable, true, true); + statement.dialect(otherSqlDialect); + m = statement.createStatement(conn, container, user); + m.close(); + m = null; + validateInsert.apply(statement); + } + } + + // Insert with vocabulary properties + { + statement = insertStatement(dataClassTable, true, true); + statement.setVocabularyProperties(vocabProps); + m = statement.createStatement(conn, container, user); + m.close(); + m = null; + assertTrue(statement.parameters.keySet().containsAll(vocabParameters)); + } + } + finally + { + if (null != m) + m.close(); + } + } + + @Test + public void testUpdate() throws Exception + { + ParameterMapStatement m = null; + try (Connection conn = getConnection()) + { + m = updateStatement(conn, principalsTable, container, user, true, true); + m.close(); m = null; + + m = updateStatement(conn, testTable, container, user, true, true); + m.close(); m = null; + } + finally + { + if (null != m) + m.close(); + } + } + + @Test + public void testUpdateWithExtensibleDomain() throws Exception + { + ParameterMapStatement m = null; + try (Connection conn = getConnection(dataClassTable)) + { + StatementUtils statement; + + // Update + { + var validateUpdate = new Function() + { + @Override + public Object apply(StatementUtils s) + { + assertTrue(s._columnTracker.insertColumns.isEmpty()); + assertFalse(s._columnTracker.updateColumns.contains("Created")); + assertFalse(s._columnTracker.updateColumns.contains("CreatedBy")); + assertTrue(s._columnTracker.updateColumns.contains("Modified")); + assertTrue(s._columnTracker.updateColumns.contains("ModifiedBy")); + assertTrue(s._columnTracker.selectColumns.isEmpty()); + var parameterKeys = s.parameters.keySet(); + assertTrue(vocabParameters.stream().noneMatch(parameterKeys::contains)); + return null; + } + }; + + statement = updateStatement(dataClassTable, true, true); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + validateUpdate.apply(statement); + + Set allUpdateColumns = new CaseInsensitiveHashSet(statement._columnTracker.updateColumns); + + if (runOtherDialect) + { + statement = updateStatement(dataClassTable, true, true); + statement.dialect(otherSqlDialect); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + validateUpdate.apply(statement); + } + + statement = updateStatement(dataClassTable, false, false); + statement.noupdate(CaseInsensitiveHashSet.of("Modified", "ModifiedBy")); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + assertFalse(statement._columnTracker.updateColumns.contains("Modified")); + assertFalse(statement._columnTracker.updateColumns.contains("ModifiedBy")); + + statement = updateStatement(dataClassTable, false, false); + statement.noupdate(allUpdateColumns); + m = statement.createStatement(conn, container, user); + var debugSql = m.getDebugSql(); + m.close(); m = null; + assertTrue(debugSql.contains("'noop' WHERE 1 <> 1")); + } + + // Update with vocabulary properties + { + statement = updateStatement(dataClassTable, true, true); + statement.setVocabularyProperties(vocabProps); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + assertTrue(statement.parameters.keySet().containsAll(vocabParameters)); + } + } + finally + { + if (null != m) + m.close(); + } + } + + @Test + public void testUpdateWithObjectUriColumn() throws Exception + { + // Arrange + // Create a list + String listName = "StatementUtilsTestList"; + { + // Create a list domain + var listDef = ListService.get().createList(container, listName, ListDefinition.KeyType.AutoIncrementInteger); + listDef.setKeyName("pk"); + + Domain domain = requireNonNull(listDef.getDomain()); + addProperty(domain, "pk", PropertyType.INTEGER); + addProperty(domain, "name", PropertyType.STRING); + + listDef.save(user); + } + + ParameterMapStatement m = null; + TableInfo listTable = requireNonNull(ListService.get().getList(container, listName)).getTable(user, container); + assertNotNull(listTable); + + try (Connection conn = getConnection(listTable)) + { + StatementUtils statement; + var expectedNoUpdateColumns = CaseInsensitiveHashSet.of("EntityId", "Created", "CreatedBy"); + var expectedUpdateColumns = CaseInsensitiveHashSet.of("DIImportHash", "LastIndexed", "Modified", "ModifiedBy", "Name"); + + // Update statement (selectIds = true, autoFillDefaultColumns = true) + { + statement = StatementUtils.updateStatement(listTable, true, true); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + + // Assert + assertEquals(expectedNoUpdateColumns, statement._dontUpdateColumnNames); + assertTrue(statement._columnTracker.insertColumns.isEmpty()); + assertTrue(statement._columnTracker.selectColumns.isEmpty()); + + if (runOtherDialect) + { + statement = StatementUtils.updateStatement(listTable, true, true); + statement.dialect(otherSqlDialect); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + } + + assertEquals(expectedNoUpdateColumns, statement._dontUpdateColumnNames); + assertTrue(statement._columnTracker.insertColumns.isEmpty()); + assertTrue(statement._columnTracker.selectColumns.isEmpty()); + + assertEquals(expectedUpdateColumns, statement._columnTracker.updateColumns); + } + + // Update statement (selectIds = false, autoFillDefaultColumns = false) + { + statement = StatementUtils.updateStatement(listTable, false, false); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + + // Assert + assertEquals(expectedNoUpdateColumns, statement._dontUpdateColumnNames); + assertTrue(statement._columnTracker.insertColumns.isEmpty()); + assertEquals(expectedUpdateColumns, statement._columnTracker.updateColumns); + + if (runOtherDialect) + { + statement = StatementUtils.updateStatement(listTable, false, false); + statement.dialect(otherSqlDialect); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + } + + assertEquals(expectedNoUpdateColumns, statement._dontUpdateColumnNames); + assertTrue(statement._columnTracker.insertColumns.isEmpty()); + assertEquals(expectedUpdateColumns, statement._columnTracker.updateColumns); + } + } + finally + { + if (null != m) + m.close(); + } + } + + @Test + public void testMerge() throws Exception + { + ParameterMapStatement m = null; + try (Connection conn = getConnection()) + { + m = mergeStatement(conn, principalsTable, null, null, null, container, user, false, true, false); + m.close(); m = null; + + if (runOtherDialect) + { + StatementUtils statement = mergeStatement(principalsTable, null, null, null, false, true, false); + statement.dialect(otherSqlDialect); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + } + + m = mergeStatement(conn, testTable, null, null, null, container, user, false, true, false); + m.close(); m = null; + + if (runOtherDialect) + { + StatementUtils statement = mergeStatement(testTable, null, null, null, false, true, false); + statement.dialect(otherSqlDialect); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + } + } + finally + { + if (null != m) + m.close(); + } + } + + @Test + public void testMergeWithExtensibleDomain() throws Exception + { + ParameterMapStatement m = null; + try (Connection conn = getConnection(dataClassTable)) + { + StatementUtils statement; + + // Merge + { + var validateMerge = new Function() + { + @Override + public Object apply(StatementUtils s) + { + boolean isPostgres = s._dialect.isPostgreSQL(); + + assertTrue(s._columnTracker.insertColumns.contains("Container")); + assertTrue(s._columnTracker.insertColumns.contains("LSID")); + assertFalse(s._columnTracker.updateColumns.contains("Container")); + assertFalse(s._columnTracker.updateColumns.contains("LSID")); + + if (isPostgres) + { + assertTrue(s._columnTracker.selectColumns.contains("_rowid_")); + assertTrue(s._columnTracker.selectColumns.contains("_$objectid$_")); + } + else + { + // Variables are not used in SQL Server + assertFalse(s._columnTracker.selectColumns.contains("_rowid_")); + assertTrue(s._columnTracker.selectColumns.contains("@_objectid_")); + } + + var parameterKeys = s.parameters.keySet(); + assertTrue(vocabParameters.stream().noneMatch(parameterKeys::contains)); + return null; + } + }; + + statement = mergeStatement(dataClassTable, null, null, null, true, true, false); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + validateMerge.apply(statement); + + var updateColumns = new CaseInsensitiveHashSet(statement._columnTracker.updateColumns); + + if (runOtherDialect) + { + statement = mergeStatement(dataClassTable, null, null, null, true, true, false); + statement.dialect(otherSqlDialect); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + validateMerge.apply(statement); + } + + // TODO: This generates a SQL parsing error in Postgres due to the reselect statement coming before the WHERE clause +// statement = mergeStatement(dataClassTable, null, CaseInsensitiveHashSet.of("RunId"), updateColumns, true, true, false); +// m = statement.createStatement(conn, container, user); +// m.close(); m = null; +// assertTrue(statement._columnTracker.updateColumns.isEmpty()); + } + + // Merge with vocabulary properties + { + statement = mergeStatement(dataClassTable, null, null, null, false, true, false); + statement.setVocabularyProperties(vocabProps); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + assertTrue(statement.parameters.keySet().containsAll(vocabParameters)); + } + } + finally + { + if (null != m) + m.close(); + } + } + + private static void addProperty(Domain d, String name, PropertyType pt) + { + DomainProperty p = d.addProperty(); + p.setName(name); + p.setPropertyURI(d.getTypeURI() + "#" + name); + p.setRangeURI(pt.getTypeUri()); + } + + private Connection getConnection() throws SQLException + { + return getConnection(principalsTable); + } + + private Connection getConnection(TableInfo table) throws SQLException + { + return table.getSchema().getScope().getConnection(); + } + } +} diff --git a/api/src/org/labkey/api/data/Table.java b/api/src/org/labkey/api/data/Table.java index ac80f5b39a5..d28580ad7a7 100644 --- a/api/src/org/labkey/api/data/Table.java +++ b/api/src/org/labkey/api/data/Table.java @@ -646,7 +646,6 @@ static void doClose(@Nullable ResultSet rs, @Nullable Statement stmt, @Nullable if (null != conn) scope.releaseConnection(conn); } - /** * return a 'clean' list of fields to update */ @@ -661,7 +660,6 @@ protected static Map _getTableData(TableInfo table, K from, return _getTableData(table, fields, insert); } - protected static Map _getTableData(TableInfo table, Map fields, boolean insert) { if (!(fields instanceof CaseInsensitiveHashMap)) @@ -678,7 +676,6 @@ protected static Map _getTableData(TableInfo table, Map _getTableData(TableInfo table, Map fields, java.sql.Timestamp date) + protected static void _insertSpecialFields(User user, TableInfo table, Map fields, SQLFragment.NowTimestamp now) { ColumnInfo col = table.getColumn(OWNER_COLUMN_NAME); if (null != col && null != user) @@ -723,7 +718,7 @@ protected static void _insertSpecialFields(User user, TableInfo table, Map fields, java.sql.Timestamp date) + protected static void _updateSpecialFields(@Nullable User user, TableInfo table, Map fields, SQLFragment.NowTimestamp now) { ColumnInfo colModifiedBy = table.getColumn(MODIFIED_BY_COLUMN_NAME); if (null != colModifiedBy && null != user) @@ -754,11 +749,11 @@ protected static void _updateSpecialFields(@Nullable User user, TableInfo table, ColumnInfo colModified = table.getColumn(MODIFIED_COLUMN_NAME); if (null != colModified) - fields.put(colModified.getName(), date); + fields.put(colModified.getName(), now); ColumnInfo colVersion = table.getVersionColumn(); if (null != colVersion && colVersion != colModified && colVersion.getJdbcType() == JdbcType.TIMESTAMP) - fields.put(colVersion.getName(), date); + fields.put(colVersion.getName(), now); } protected static void _copyUpdateSpecialFields(TableInfo table, Object returnObject, Map fields) @@ -821,22 +816,19 @@ public static K insert(@Nullable User user, TableInfo table, K fieldsIn) { assert assertInDb(table); - // _executeTriggers(table, fields); - SQLFragment insertSQL = new SQLFragment(); SQLFragment columnSQL = new SQLFragment(); SQLFragment valueSQL = new SQLFragment(); ColumnInfo autoIncColumn = null; - ColumnInfo versionColumn = null; String comma = ""; //noinspection unchecked Map fields = fieldsIn instanceof Map ? _getTableData(table, (Map)fieldsIn, true) : _getTableData(table, fieldsIn, true); - java.sql.Timestamp date = new java.sql.Timestamp(System.currentTimeMillis()); - _insertSpecialFields(user, table, fields, date); - _updateSpecialFields(user, table, fields, date); + SQLFragment.NowTimestamp now = new SQLFragment.NowTimestamp(); + _insertSpecialFields(user, table, fields, now); + _updateSpecialFields(user, table, fields, now); List columns = table.getColumns(); @@ -868,6 +860,8 @@ else if (column.isAutoIncrement()) valueSQL.append(comma); if (null == value || value instanceof String s && s.isEmpty()) valueSQL.append("NULL"); + else if (value instanceof SQLFragment.NowTimestamp ts) + valueSQL.appendValue(ts); else { // Validate the value @@ -937,7 +931,7 @@ else if (column.isAutoIncrement()) _copyInsertSpecialFields(returnObject, fields); _copyUpdateSpecialFields(table, returnObject, fields); } - catch(SQLException e) + catch (SQLException e) { logException(insertSQL, conn, e, Level.WARN); throw new RuntimeSQLException(e); @@ -966,8 +960,6 @@ public static K update(@Nullable User user, TableInfo table, K fieldsIn, @Nu assert assertInDb(table); assert null != pkVals; - // _executeTriggers(table, previous, fields); - SQLFragment setSQL = new SQLFragment(); SQLFragment whereSQL = new SQLFragment(); String comma = ""; @@ -1028,8 +1020,7 @@ else if (pkVals instanceof Map) Map fields = fieldsIn instanceof Map ? _getTableData(table, (Map)fieldsIn, true) : _getTableData(table, fieldsIn, true); - java.sql.Timestamp date = new java.sql.Timestamp(System.currentTimeMillis()); - _updateSpecialFields(user, table, fields, date); + _updateSpecialFields(user, table, fields, new SQLFragment.NowTimestamp()); List columns = table.getColumns(); ColumnInfo colModified = table.getColumn(MODIFIED_COLUMN_NAME); @@ -1069,11 +1060,12 @@ else if (pkVals instanceof Map) Object value = fields.get(column.getName()); setSQL.append(comma); setSQL.appendIdentifier(column.getSelectIdentifier()); + setSQL.append("="); if (null == value || value instanceof String s && s.isEmpty()) - { - setSQL.append("=NULL"); - } + setSQL.append("NULL"); + else if (value instanceof SQLFragment.NowTimestamp ts) + setSQL.appendValue(ts); else { // Validate the value @@ -1085,7 +1077,7 @@ else if (pkVals instanceof Map) throw new RuntimeValidationException(msg, column.getName()); // CONSIDER: would prefer throwing ValidationException instead, but it's not a RuntimeException } - setSQL.append("=?"); + setSQL.append("?"); if (value instanceof Parameter.JdbcParameterValue) setSQL.add(value); else diff --git a/api/src/org/labkey/api/dataiterator/SimpleTranslator.java b/api/src/org/labkey/api/dataiterator/SimpleTranslator.java index a015956b15c..803c2663799 100644 --- a/api/src/org/labkey/api/dataiterator/SimpleTranslator.java +++ b/api/src/org/labkey/api/dataiterator/SimpleTranslator.java @@ -48,6 +48,7 @@ import org.labkey.api.data.LookupResolutionType; import org.labkey.api.data.MultiValuedForeignKey; import org.labkey.api.data.MvUtil; +import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.TableDescription; import org.labkey.api.data.TableInfo; @@ -1056,7 +1057,7 @@ private class TimestampColumn implements Supplier public Object get() { if (null == _ts) - _ts = new NowTimestamp(System.currentTimeMillis()); + _ts = new SQLFragment.NowTimestamp(); return _ts; } } @@ -1085,8 +1086,8 @@ public Object get() // shared tables should be Integer->Integer Integer valueAsInt = null; - if (value instanceof String) - valueAsInt = Integer.parseInt((String)value); + if (value instanceof String stringValue) + valueAsInt = Integer.parseInt(stringValue); if (_dataspaceTableIdMap.containsKey(valueAsInt)) { value = _dataspaceTableIdMap.get(valueAsInt); @@ -1979,12 +1980,12 @@ public boolean isConstant(int i) Supplier c = _outputColumns.get(i).getValue(); if (c instanceof ConstantColumn) return true; - if (c instanceof PassthroughColumn) - return _data.isConstant(((PassthroughColumn)c).index); - if (c instanceof AliasColumn) - return isConstant(((AliasColumn)c).index); - if (c instanceof SimpleConvertColumn) - return _data.isConstant(((SimpleConvertColumn)c).index); + if (c instanceof PassthroughColumn pc) + return _data.isConstant(pc.index); + if (c instanceof AliasColumn ac) + return isConstant(ac.index); + if (c instanceof SimpleConvertColumn scc) + return _data.isConstant(scc.index); if (c instanceof TimestampColumn) return true; return false; @@ -1995,18 +1996,16 @@ public boolean isConstant(int i) public Object getConstantValue(int i) { Supplier c = _outputColumns.get(i).getValue(); - if (c instanceof ConstantColumn) - return ((ConstantColumn)c).k; - if (c instanceof PassthroughColumn) - return _data.getConstantValue(((PassthroughColumn)c).index); - if (c instanceof AliasColumn) - return getConstantValue(((AliasColumn)c).index); + if (c instanceof ConstantColumn cc) + return cc.k; + if (c instanceof PassthroughColumn pc) + return _data.getConstantValue(pc.index); + if (c instanceof AliasColumn ac) + return getConstantValue(ac.index); if (c instanceof SimpleConvertColumn scc) - { return scc.convert(_data.getConstantValue(scc.index)); - } if (c instanceof TimestampColumn) - return new NowTimestamp(System.currentTimeMillis()); + return new SQLFragment.NowTimestamp(); throw new IllegalStateException("shouldn't call this method unless isConstant()==true"); } @@ -2017,17 +2016,6 @@ public void close() throws IOException _data.close(); } - - // this is a marker interface to hint that this value may be replaced by {ts now()} - public static class NowTimestamp extends java.sql.Timestamp - { - public NowTimestamp(long ms) - { - super(ms); - } - } - - @Override public void debugLogInfo(StringBuilder sb) { diff --git a/audit/src/org/labkey/audit/AuditLogImpl.java b/audit/src/org/labkey/audit/AuditLogImpl.java index 810d90ee07d..60d33f08858 100644 --- a/audit/src/org/labkey/audit/AuditLogImpl.java +++ b/audit/src/org/labkey/audit/AuditLogImpl.java @@ -1,327 +1,316 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.audit; - -import jakarta.servlet.ServletContext; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.AuditTypeProvider; -import org.labkey.api.audit.DetailedAuditTypeEvent; -import org.labkey.api.audit.SampleTimelineAuditEvent; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Sort; -import org.labkey.api.data.TableSelector; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.UserSchema; -import org.labkey.api.security.User; -import org.labkey.api.security.UserManager; -import org.labkey.api.util.ContextListener; -import org.labkey.api.util.Pair; -import org.labkey.api.util.StartupListener; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.HttpView; -import org.labkey.audit.model.LogManager; -import org.labkey.audit.query.AuditQuerySchema; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - -/** - * User: Karl Lum - * Date: Oct 4, 2007 - */ -public class AuditLogImpl implements AuditLogService, StartupListener -{ - private static final AuditLogImpl _instance = new AuditLogImpl(); - - private static final Logger _log = LogHelper.getLogger(AuditLogImpl.class, "Audit service interactions."); - - private final Queue> _eventTypeQueue = new LinkedList<>(); - private final AtomicBoolean _logToDatabase = new AtomicBoolean(false); - private static final Object STARTUP_LOCK = new Object(); - - // Cache the audit events associated with transaction ids. We currently use these for interacting with objects - // that were created immediately after they were created, so the cache size does not need to be very large and the defaultTimeToLive can be small. - // Use a pair as the cache object to avoid warnings about mutable cache objects (Issue 48779). - // Since this is all about capturing data from the same transaction, there shouldn't be other threads in the mix. - private static final Cache>> TRANSACTION_EVENT_CACHE = CacheManager.getBlockingCache(50, CacheManager.HOUR, - "Transaction Audit Event Cache", - (key, argument) -> Pair.of(key, new ArrayList<>()) - ); - - public static AuditLogImpl get() - { - return _instance; - } - - private AuditLogImpl() - { - // If we're migrating, avoid creating all the audit log tables and inserting the queued events - if (ModuleLoader.getInstance().shouldInsertData()) - ContextListener.addStartupListener(this); - } - - @Override - public String getName() - { - return "Audit Log"; - } - - @Override - public void moduleStartupComplete(ServletContext servletContext) - { - // perform audit provider initialization - for (AuditTypeProvider provider : AuditLogService.get().getAuditProviders()) - { - provider.initializeProvider(User.getAdminServiceUser()); - } - - // Synchronize so that we can guarantee that all events have already been added to the queue before we - // start processing them - synchronized (STARTUP_LOCK) - { - _logToDatabase.set(true); - } - - while (!_eventTypeQueue.isEmpty()) - { - Pair event = _eventTypeQueue.remove(); - addEvents(event.first, List.of(event.second)); - } - } - - @Override - public boolean isViewable() - { - return true; - } - - @Override - public K addEvent(User user, K event) - { - return _addEvents(user, List.of(event),true, false); - } - - @Override - public void addEvents(@Nullable User user, List events) - { - _addEvents(user, events, false, false); - } - - @Override - public void addEvents(@Nullable User user, List events, boolean useTransactionAuditCache) - { - _addEvents(user, events, false, useTransactionAuditCache); - } - - private K _addEvents(@Nullable User user, List events, boolean reselectEvent, boolean useTransactionAuditCache) - { - assert !reselectEvent || events.size() == 1; - - for (var event : events) - { - assert event.getContainer() != null : "Container cannot be null"; - - if (user == null) - { - if (HttpView.hasCurrentView() && HttpView.currentContext() != null) - _log.warn("user was not specified for event type " + event.getEventType() + " in container " + event.getContainer() + "; defaulting to guest user."); - user = UserManager.getGuestUser(); - } - if (event.getTransactionId() != null && useTransactionAuditCache) - { - List transactionEvents = TRANSACTION_EVENT_CACHE.get(event.getTransactionId()).second; - transactionEvents.add(event); - } - - // ensure some standard fields - if (event.getCreated() == null) - event.setCreated(new Date()); - if (event.getCreatedBy() == null) - event.setCreatedBy(user); - - if (event.getImpersonatedBy() == null && user.isImpersonated()) - { - User impersonatingUser = user.getImpersonatingUser(); - event.setImpersonatedBy(impersonatingUser.getUserId()); - } - } - - try (var ignored = SpringActionController.ignoreSqlUpdates()) - { - /* - This is necessary because audit log service needs to be registered in the constructor - of the audit module, but the schema may not be created or updated at that point. Events - that occur before startup is complete are therefore queued up and recorded after startup. - */ - boolean databaseReady; - synchronized (STARTUP_LOCK) - { - // Keep the critical section as lean as possible - just guarantee that all the events - // have been queued before releasing the lock - databaseReady = _logToDatabase.get(); - if (!databaseReady) - { - for (var event : events) - _eventTypeQueue.add(new Pair<>(user, event)); - } - } - - if (databaseReady) - { - if (reselectEvent && events.size()==1) - return LogManager.get().insertEvent(user, events.get(0)); - LogManager.get().insertEvents(user, events); - } - } - catch (RuntimeException e) - { - _log.error("Failed to insert audit log event", e); - AuditLogService.handleAuditFailure(user, e); - throw e; - } - return null; - } - - @Override - public UserSchema createSchema(User user, Container container) - { - return new AuditQuerySchema(user, container); - } - - @Nullable - @Override - public K getAuditEvent(User user, String eventType, int rowId) - { - return LogManager.get().getAuditEvent(user, eventType, rowId); - } - - @Nullable - @Override - public K getAuditEvent(User user, String eventType, int rowId, @Nullable ContainerFilter cf) - { - return LogManager.get().getAuditEvent(user, eventType, rowId, cf); - } - - @Override - public List getAuditEvents(Container container, User user, String eventType, @Nullable SimpleFilter filter, @Nullable Sort sort) - { - return LogManager.get().getAuditEvents(container, user, eventType, filter, sort); - } - - @Override - public List getAuditEvents(Container container, User user, String eventType, @Nullable SimpleFilter filter, @Nullable Sort sort, @Nullable ContainerFilter cf) - { - return LogManager.get().getAuditEvents(container, user, eventType, filter, sort, cf); - } - - @Override - public ActionURL getAuditUrl() - { - return new ActionURL(AuditController.ShowAuditLogAction.class, ContainerManager.getRoot()); - } - - public List getTransactionSampleIds(long transactionAuditId, User user, Container container, @Nullable ContainerFilter containerFilter) - { - List transactionEvents = TRANSACTION_EVENT_CACHE.get(transactionAuditId).second; - if (!transactionEvents.isEmpty()) - { - List ids = new ArrayList<>(); - transactionEvents.forEach(event -> { - if (event instanceof SampleTimelineAuditEvent stEvent) - ids.add(stEvent.getSampleId()); - }); - return ids; - } - - SimpleFilter filter = new SimpleFilter(); - filter.addCondition(FieldKey.fromParts("TransactionID"), transactionAuditId); - - List events = AuditLogService.get().getAuditEvents(container, user, SampleTimelineAuditEvent.EVENT_TYPE, filter, null, containerFilter); - return events.stream().map(SampleTimelineAuditEvent::getSampleId).collect(Collectors.toList()); - } - - public List getTransactionSourceIds(long transactionAuditId, User user, Container container, @Nullable ContainerFilter containerFilter) - { - List lsids = new ArrayList<>(); - List sourceIds = new ArrayList<>(); - List transactionEvents = TRANSACTION_EVENT_CACHE.get(transactionAuditId).second; - if (!transactionEvents.isEmpty()) - { - transactionEvents.forEach(event -> { - if (event instanceof DetailedAuditTypeEvent detailedEvent) - { - if (detailedEvent.getNewRecordMap() != null) - { - Map newRecord = new CaseInsensitiveHashMap<>(AbstractAuditTypeProvider.decodeFromDataMap(detailedEvent.getNewRecordMap())); - if (newRecord.containsKey("RowId") && !StringUtils.isEmpty(newRecord.get("RowId"))) - sourceIds.add(Long.valueOf(newRecord.get("RowId"))); - else if (newRecord.containsKey("LSID") && !StringUtils.isEmpty(newRecord.get("LSID"))) - lsids.add(newRecord.get("LSID")); - } - } - }); - } - else - { - List events = QueryService.get().getQueryUpdateAuditRecords(user, container, transactionAuditId, containerFilter); - - events.forEach((event) -> { - if (event.getNewRecordMap() != null) - { - Map newRecord = new CaseInsensitiveHashMap<>(AbstractAuditTypeProvider.decodeFromDataMap(event.getNewRecordMap())); - if (newRecord.containsKey("RowId") && !StringUtils.isEmpty(newRecord.get("RowId"))) - sourceIds.add(Long.valueOf(newRecord.get("RowId"))); - else if (newRecord.containsKey("LSID") && !StringUtils.isEmpty(newRecord.get("LSID"))) - lsids.add(newRecord.get("LSID")); - - } - }); - } - if (!lsids.isEmpty()) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromParts("LSID"), lsids, CompareType.IN); - TableSelector selector = new TableSelector(ExperimentService.get().getTinfoData(), Collections.singleton("RowId"), filter, null); - sourceIds.addAll(selector.getArrayList(Long.class)); - } - return sourceIds; - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.audit; + +import jakarta.servlet.ServletContext; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.AuditTypeProvider; +import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.SampleTimelineAuditEvent; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; +import org.labkey.api.data.TableSelector; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.util.ContextListener; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StartupListener; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.HttpView; +import org.labkey.audit.model.LogManager; +import org.labkey.audit.query.AuditQuerySchema; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +public class AuditLogImpl implements AuditLogService, StartupListener +{ + private static final AuditLogImpl _instance = new AuditLogImpl(); + + private static final Logger _log = LogHelper.getLogger(AuditLogImpl.class, "Audit service interactions."); + + private final Queue> _eventTypeQueue = new LinkedList<>(); + private final AtomicBoolean _logToDatabase = new AtomicBoolean(false); + private static final Object STARTUP_LOCK = new Object(); + + // Cache the audit events associated with transaction ids. We currently use these for interacting with objects + // that were created immediately after they were created, so the cache size does not need to be very large and the defaultTimeToLive can be small. + // Use a pair as the cache object to avoid warnings about mutable cache objects (Issue 48779). + // Since this is all about capturing data from the same transaction, there shouldn't be other threads in the mix. + private static final Cache>> TRANSACTION_EVENT_CACHE = CacheManager.getBlockingCache(50, CacheManager.HOUR, + "Transaction Audit Event Cache", + (key, argument) -> Pair.of(key, new ArrayList<>()) + ); + + public static AuditLogImpl get() + { + return _instance; + } + + private AuditLogImpl() + { + // If we're migrating, avoid creating all the audit log tables and inserting the queued events + if (ModuleLoader.getInstance().shouldInsertData()) + ContextListener.addStartupListener(this); + } + + @Override + public String getName() + { + return "Audit Log"; + } + + @Override + public void moduleStartupComplete(ServletContext servletContext) + { + // perform audit provider initialization + for (AuditTypeProvider provider : AuditLogService.get().getAuditProviders()) + { + provider.initializeProvider(User.getAdminServiceUser()); + } + + // Synchronize so that we can guarantee that all events have already been added to the queue before we + // start processing them + synchronized (STARTUP_LOCK) + { + _logToDatabase.set(true); + } + + while (!_eventTypeQueue.isEmpty()) + { + Pair event = _eventTypeQueue.remove(); + addEvents(event.first, List.of(event.second)); + } + } + + @Override + public boolean isViewable() + { + return true; + } + + @Override + public K addEvent(User user, K event) + { + return _addEvents(user, List.of(event),true, false); + } + + @Override + public void addEvents(@Nullable User user, List events) + { + _addEvents(user, events, false, false); + } + + @Override + public void addEvents(@Nullable User user, List events, boolean useTransactionAuditCache) + { + _addEvents(user, events, false, useTransactionAuditCache); + } + + private K _addEvents(@Nullable User user, List events, boolean reselectEvent, boolean useTransactionAuditCache) + { + assert !reselectEvent || events.size() == 1; + + for (var event : events) + { + assert event.getContainer() != null : "Container cannot be null"; + + if (user == null) + { + if (HttpView.hasCurrentView() && HttpView.currentContext() != null) + _log.warn("user was not specified for event type " + event.getEventType() + " in container " + event.getContainer() + "; defaulting to guest user."); + user = UserManager.getGuestUser(); + } + if (event.getTransactionId() != null && useTransactionAuditCache) + { + List transactionEvents = TRANSACTION_EVENT_CACHE.get(event.getTransactionId()).second; + transactionEvents.add(event); + } + + if (event.getImpersonatedBy() == null && user.isImpersonated()) + { + User impersonatingUser = user.getImpersonatingUser(); + event.setImpersonatedBy(impersonatingUser.getUserId()); + } + } + + try (var ignored = SpringActionController.ignoreSqlUpdates()) + { + /* + This is necessary because audit log service needs to be registered in the constructor + of the audit module, but the schema may not be created or updated at that point. Events + that occur before startup is complete are therefore queued up and recorded after startup. + */ + boolean databaseReady; + synchronized (STARTUP_LOCK) + { + // Keep the critical section as lean as possible - just guarantee that all the events + // have been queued before releasing the lock + databaseReady = _logToDatabase.get(); + if (!databaseReady) + { + for (var event : events) + _eventTypeQueue.add(new Pair<>(user, event)); + } + } + + if (databaseReady) + { + if (reselectEvent && events.size()==1) + return LogManager.get().insertEvent(user, events.get(0)); + LogManager.get().insertEvents(user, events); + } + } + catch (RuntimeException e) + { + _log.error("Failed to insert audit log event", e); + AuditLogService.handleAuditFailure(user, e); + throw e; + } + return null; + } + + @Override + public UserSchema createSchema(User user, Container container) + { + return new AuditQuerySchema(user, container); + } + + @Nullable + @Override + public K getAuditEvent(User user, String eventType, int rowId) + { + return LogManager.get().getAuditEvent(user, eventType, rowId); + } + + @Nullable + @Override + public K getAuditEvent(User user, String eventType, int rowId, @Nullable ContainerFilter cf) + { + return LogManager.get().getAuditEvent(user, eventType, rowId, cf); + } + + @Override + public List getAuditEvents(Container container, User user, String eventType, @Nullable SimpleFilter filter, @Nullable Sort sort) + { + return LogManager.get().getAuditEvents(container, user, eventType, filter, sort); + } + + @Override + public List getAuditEvents(Container container, User user, String eventType, @Nullable SimpleFilter filter, @Nullable Sort sort, @Nullable ContainerFilter cf) + { + return LogManager.get().getAuditEvents(container, user, eventType, filter, sort, cf); + } + + @Override + public ActionURL getAuditUrl() + { + return new ActionURL(AuditController.ShowAuditLogAction.class, ContainerManager.getRoot()); + } + + public List getTransactionSampleIds(long transactionAuditId, User user, Container container, @Nullable ContainerFilter containerFilter) + { + List transactionEvents = TRANSACTION_EVENT_CACHE.get(transactionAuditId).second; + if (!transactionEvents.isEmpty()) + { + List ids = new ArrayList<>(); + transactionEvents.forEach(event -> { + if (event instanceof SampleTimelineAuditEvent stEvent) + ids.add(stEvent.getSampleId()); + }); + return ids; + } + + SimpleFilter filter = new SimpleFilter(); + filter.addCondition(FieldKey.fromParts("TransactionID"), transactionAuditId); + + List events = AuditLogService.get().getAuditEvents(container, user, SampleTimelineAuditEvent.EVENT_TYPE, filter, null, containerFilter); + return events.stream().map(SampleTimelineAuditEvent::getSampleId).collect(Collectors.toList()); + } + + public List getTransactionSourceIds(long transactionAuditId, User user, Container container, @Nullable ContainerFilter containerFilter) + { + List lsids = new ArrayList<>(); + List sourceIds = new ArrayList<>(); + List transactionEvents = TRANSACTION_EVENT_CACHE.get(transactionAuditId).second; + if (!transactionEvents.isEmpty()) + { + transactionEvents.forEach(event -> { + if (event instanceof DetailedAuditTypeEvent detailedEvent) + { + if (detailedEvent.getNewRecordMap() != null) + { + Map newRecord = new CaseInsensitiveHashMap<>(AbstractAuditTypeProvider.decodeFromDataMap(detailedEvent.getNewRecordMap())); + if (newRecord.containsKey("RowId") && !StringUtils.isEmpty(newRecord.get("RowId"))) + sourceIds.add(Long.valueOf(newRecord.get("RowId"))); + else if (newRecord.containsKey("LSID") && !StringUtils.isEmpty(newRecord.get("LSID"))) + lsids.add(newRecord.get("LSID")); + } + } + }); + } + else + { + List events = QueryService.get().getQueryUpdateAuditRecords(user, container, transactionAuditId, containerFilter); + + events.forEach((event) -> { + if (event.getNewRecordMap() != null) + { + Map newRecord = new CaseInsensitiveHashMap<>(AbstractAuditTypeProvider.decodeFromDataMap(event.getNewRecordMap())); + if (newRecord.containsKey("RowId") && !StringUtils.isEmpty(newRecord.get("RowId"))) + sourceIds.add(Long.valueOf(newRecord.get("RowId"))); + else if (newRecord.containsKey("LSID") && !StringUtils.isEmpty(newRecord.get("LSID"))) + lsids.add(newRecord.get("LSID")); + + } + }); + } + if (!lsids.isEmpty()) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromParts("LSID"), lsids, CompareType.IN); + TableSelector selector = new TableSelector(ExperimentService.get().getTinfoData(), Collections.singleton("RowId"), filter, null); + sourceIds.addAll(selector.getArrayList(Long.class)); + } + return sourceIds; + } +} diff --git a/audit/src/org/labkey/audit/model/LogManager.java b/audit/src/org/labkey/audit/model/LogManager.java index 0f64bba8367..1ee60f5b965 100644 --- a/audit/src/org/labkey/audit/model/LogManager.java +++ b/audit/src/org/labkey/audit/model/LogManager.java @@ -1,294 +1,289 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.audit.model; - -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.AuditTypeProvider; -import org.labkey.api.audit.query.DefaultAuditTypeTable; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.ObjectFactory; -import org.labkey.api.data.ParameterMapStatement; -import org.labkey.api.data.PropertyStorageSpec; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Sort; -import org.labkey.api.data.StatementUtils; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainKind; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.query.UserSchema; -import org.labkey.api.security.User; -import org.labkey.api.view.HttpView; -import org.labkey.audit.AuditSchema; - -import java.sql.Connection; -import java.sql.SQLException; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; - - -/** - * User: Karl Lum - * Date: Oct 4, 2007 - */ -public class LogManager -{ - private static final Logger _log = org.apache.logging.log4j.LogManager.getLogger(LogManager.class); - private static final LogManager _instance = new LogManager(); - - private LogManager(){} - static public LogManager get() - { - return _instance; - } - - public DbSchema getSchema() - { - return AuditSchema.getInstance().getSchema(); - } - - /** There are a few places that depend on the reselect behavior. e.g. to get the rowid of the event */ - public K insertEvent(User user, K type) - { - Logger auditLogger = getAuditLogger(type); - auditLogger.info(type.getAuditLogMessage()); - - AuditTypeProvider provider = AuditLogService.get().getAuditProvider(type.getEventType()); - - if (provider != null) - { - Container c = type.getContainer(); - - UserSchema schema = AuditLogService.getAuditLogSchema(user, c != null ? c : ContainerManager.getRoot()); - - if (schema != null) - { - TableInfo table = schema.getTable(provider.getEventName(), false); - - if (table instanceof DefaultAuditTypeTable) - { - // consider using etl data iterator for inserts - type = validateFields(provider, type); - TableInfo dbTable = ((DefaultAuditTypeTable)table).getRealTable(); - K ret = Table.insert(user, dbTable, type); - return ret; - } - } - } - return null; - } - - private static Logger getAuditLogger(K type) - { - return org.apache.logging.log4j.LogManager.getLogger("org.labkey.audit.event." + type.getEventType().replaceAll(" ", "")); - } - - /** all events must be of same event type and container, for optimized code path */ - public void insertEvents(User user, List events) - { - if (events.isEmpty()) - return; - - AuditTypeEvent type = events.get(0); - - // Out of an abundance of caution and backward compatible behavior, do one-at-a-time logging if - // there is no transaction. Can revisit if this is not necessary. - // Keep in mind that the audit schema might not be in the same scope as table that is being logged about. - boolean optimize = events.size() == 1 || getSchema().getScope().isTransactionActive(); - if (optimize) - { - // make sure all events are the same type - final String expectedEventType = type.getEventType(); - final Container expectedContainer = type.getContainer(); - Optional problemEvent = events.stream() - .filter(event -> !Objects.equals(expectedEventType, event.getEventType()) || !Objects.equals(expectedContainer, event.getContainer())) - .findAny(); - optimize = problemEvent.isEmpty(); - } - - if (!optimize) - { - // do one at a time if events are not all the same - for (var event : events) - insertEvent(user, event); - return; - } - - AuditTypeProvider provider = AuditLogService.get().getAuditProvider(type.getEventType()); - if (null == provider) - return; - Container c = type.getContainer(); - UserSchema schema = AuditLogService.getAuditLogSchema(user, c != null ? c : ContainerManager.getRoot()); - TableInfo table = null==schema ? null : schema.getTable(provider.getEventName(), false); - TableInfo dbTable = table instanceof DefaultAuditTypeTable ? ((DefaultAuditTypeTable) table).getRealTable() : null; - - Logger auditLogger = getAuditLogger(type); - SQLException sqlx = null; - - if (null != dbTable) - { - try (Connection conn = dbTable.getSchema().getScope().getConnection()) - { - ParameterMapStatement stmt = StatementUtils.insertStatement(conn, dbTable, c, user, false, true); - for (var event : events) - { - event = validateFields(provider, event); - Map map = ObjectFactory.Registry.getFactory((Class)event.getClass()).toMap(event, null); - stmt.clearParameters(); - stmt.putAll(map); - stmt.addBatch(); - } - stmt.executeBatch(); - } - catch (SQLException x) - { - auditLogger.warn("Error occurred saving audit entries to database"); - sqlx = x; - } - } - - if (auditLogger.isInfoEnabled()) - { - // CONSIDER log these in TX.addCommitTask()? (but then what if updates are happening in a different scope?) - for (var event : events) - auditLogger.info(event.getAuditLogMessage()); - } - - if (null != sqlx) - throw new RuntimeSQLException(sqlx); - } - - @Nullable - public K getAuditEvent(User user, String eventType, int rowId, @Nullable ContainerFilter cf) - { - AuditTypeProvider provider = AuditLogService.get().getAuditProvider(eventType); - if (provider != null) - { - UserSchema schema = AuditLogService.getAuditLogSchema(user, HttpView.currentContext().getContainer()); - - if (schema != null) - { - TableInfo table = schema.getTable(provider.getEventName(), cf); - TableSelector selector = new TableSelector(table, null, null); - - return selector.getObject(rowId, provider.getEventClass()); - } - } - return null; - } - - @Nullable - public K getAuditEvent(User user, String eventType, int rowId) - { - return getAuditEvent(user, eventType, rowId, null); - } - - public List getAuditEvents(Container container, User user, String eventType, @Nullable SimpleFilter filter, @Nullable Sort sort) - { - return getAuditEvents(container, user, eventType, filter, sort, null); - } - public List getAuditEvents(Container container, User user, String eventType, @Nullable SimpleFilter filter, @Nullable Sort sort, @Nullable ContainerFilter cf) - { - AuditTypeProvider provider = AuditLogService.get().getAuditProvider(eventType); - if (provider != null) - { - UserSchema schema = AuditLogService.getAuditLogSchema(user, container); - - if (schema != null) - { - TableInfo table = schema.getTable(provider.getEventName(), cf); - TableSelector selector = new TableSelector(table, filter, sort); - - return selector.getArrayList(provider.getEventClass()); - } - } - return Collections.emptyList(); - } - - /** - * Ensure that the string properties don't exceed the length of the provisioned columns. - * Values will be trimmed to the max length. - */ - private K validateFields(@NotNull AuditTypeProvider provider, @NotNull K type) - { - ObjectFactory factory = ObjectFactory.Registry.getFactory((Class)type.getClass()); - Map values = new CaseInsensitiveHashMap<>(); - factory.toMap(type, values); - - boolean changed = false; - Domain domain = provider.getDomain(); - - DomainKind domainKind = domain.getDomainKind(); - for (PropertyStorageSpec prop : domainKind.getBaseProperties(domain)) - { - Object value = values.get(prop.getName()); - if (prop.getJdbcType().isText() && value instanceof String s) - { - int scale = prop.getSize(); - if (s.length() > scale) - { - _log.warn("Audit field input : \n" + prop.getName() + "\nexceeded the maximum length : " + scale); - String trimmed = s.substring(0, scale-3) + "..."; - values.put(prop.getName(), trimmed); - changed = true; - } - } - } - - for (DomainProperty dp : domain.getProperties()) - { - // For now, only check for string length like we were doing for the old audit event fields - PropertyDescriptor pd = dp.getPropertyDescriptor(); - Object value = values.get(dp.getName()); - if (pd.isStringType() && value instanceof String s) - { - int scale = dp.getScale(); - if (scale > 0 && s.length() > scale) - { - _log.warn("Audit field input : \n" + pd.getName() + "\nexceeded the maximum length : " + scale); - String trimmed; - if (scale > 100) - trimmed = s.substring(0, scale-3) + "..."; - else - trimmed = s.substring(0, scale); - values.put(pd.getName(), trimmed); - changed = true; - } - } - } - - if (changed) - return factory.fromMap(values); - else - return type; - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.audit.model; + +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.AuditTypeProvider; +import org.labkey.api.audit.query.DefaultAuditTypeTable; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.ObjectFactory; +import org.labkey.api.data.ParameterMapStatement; +import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; +import org.labkey.api.data.StatementUtils; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainKind; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; +import org.labkey.api.view.HttpView; +import org.labkey.audit.AuditSchema; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +public class LogManager +{ + private static final Logger _log = org.apache.logging.log4j.LogManager.getLogger(LogManager.class); + private static final LogManager _instance = new LogManager(); + + private LogManager(){} + static public LogManager get() + { + return _instance; + } + + public DbSchema getSchema() + { + return AuditSchema.getInstance().getSchema(); + } + + /** There are a few places that depend on the reselect behavior. e.g. to get the rowid of the event */ + public K insertEvent(User user, K type) + { + Logger auditLogger = getAuditLogger(type); + auditLogger.info(type.getAuditLogMessage()); + + AuditTypeProvider provider = AuditLogService.get().getAuditProvider(type.getEventType()); + + if (provider != null) + { + Container c = type.getContainer(); + + UserSchema schema = AuditLogService.getAuditLogSchema(user, c != null ? c : ContainerManager.getRoot()); + + if (schema != null) + { + TableInfo table = schema.getTable(provider.getEventName(), false); + + if (table instanceof DefaultAuditTypeTable auditTypeTable) + { + // consider using etl data iterator for inserts + type = validateFields(provider, type); + TableInfo dbTable = auditTypeTable.getRealTable(); + K ret = Table.insert(user, dbTable, type); + return ret; + } + } + } + return null; + } + + private static Logger getAuditLogger(K type) + { + return org.apache.logging.log4j.LogManager.getLogger("org.labkey.audit.event." + type.getEventType().replaceAll(" ", "")); + } + + /** all events must be of same event type and container, for optimized code path */ + public void insertEvents(User user, List events) + { + if (events.isEmpty()) + return; + + AuditTypeEvent type = events.get(0); + + // Out of an abundance of caution and backward compatible behavior, do one-at-a-time logging if + // there is no transaction. Can revisit if this is not necessary. + // Keep in mind that the audit schema might not be in the same scope as table that is being logged about. + boolean optimize = events.size() == 1 || getSchema().getScope().isTransactionActive(); + if (optimize) + { + // make sure all events are the same type + final String expectedEventType = type.getEventType(); + final Container expectedContainer = type.getContainer(); + Optional problemEvent = events.stream() + .filter(event -> !Objects.equals(expectedEventType, event.getEventType()) || !Objects.equals(expectedContainer, event.getContainer())) + .findAny(); + optimize = problemEvent.isEmpty(); + } + + if (!optimize) + { + // do one at a time if events are not all the same + for (var event : events) + insertEvent(user, event); + return; + } + + AuditTypeProvider provider = AuditLogService.get().getAuditProvider(type.getEventType()); + if (null == provider) + return; + Container c = type.getContainer(); + UserSchema schema = AuditLogService.getAuditLogSchema(user, c != null ? c : ContainerManager.getRoot()); + TableInfo table = null==schema ? null : schema.getTable(provider.getEventName(), false); + TableInfo dbTable = table instanceof DefaultAuditTypeTable auditTypeTable ? auditTypeTable.getRealTable() : null; + + Logger auditLogger = getAuditLogger(type); + SQLException sqlx = null; + + if (null != dbTable) + { + try (Connection conn = dbTable.getSchema().getScope().getConnection()) + { + ParameterMapStatement stmt = StatementUtils.insertStatement(conn, dbTable, c, user, false, true); + for (var event : events) + { + event = validateFields(provider, event); + Map map = ObjectFactory.Registry.getFactory((Class)event.getClass()).toMap(event, null); + stmt.clearParameters(); + stmt.putAll(map); + stmt.addBatch(); + } + stmt.executeBatch(); + } + catch (SQLException x) + { + auditLogger.warn("Error occurred saving audit entries to database"); + sqlx = x; + } + } + + if (auditLogger.isInfoEnabled()) + { + // CONSIDER log these in TX.addCommitTask()? (but then what if updates are happening in a different scope?) + for (var event : events) + auditLogger.info(event.getAuditLogMessage()); + } + + if (null != sqlx) + throw new RuntimeSQLException(sqlx); + } + + @Nullable + public K getAuditEvent(User user, String eventType, int rowId, @Nullable ContainerFilter cf) + { + AuditTypeProvider provider = AuditLogService.get().getAuditProvider(eventType); + if (provider != null) + { + UserSchema schema = AuditLogService.getAuditLogSchema(user, HttpView.currentContext().getContainer()); + + if (schema != null) + { + TableInfo table = schema.getTable(provider.getEventName(), cf); + TableSelector selector = new TableSelector(table, null, null); + + return selector.getObject(rowId, provider.getEventClass()); + } + } + return null; + } + + @Nullable + public K getAuditEvent(User user, String eventType, int rowId) + { + return getAuditEvent(user, eventType, rowId, null); + } + + public List getAuditEvents(Container container, User user, String eventType, @Nullable SimpleFilter filter, @Nullable Sort sort) + { + return getAuditEvents(container, user, eventType, filter, sort, null); + } + public List getAuditEvents(Container container, User user, String eventType, @Nullable SimpleFilter filter, @Nullable Sort sort, @Nullable ContainerFilter cf) + { + AuditTypeProvider provider = AuditLogService.get().getAuditProvider(eventType); + if (provider != null) + { + UserSchema schema = AuditLogService.getAuditLogSchema(user, container); + + if (schema != null) + { + TableInfo table = schema.getTable(provider.getEventName(), cf); + TableSelector selector = new TableSelector(table, filter, sort); + + return selector.getArrayList(provider.getEventClass()); + } + } + return Collections.emptyList(); + } + + /** + * Ensure that the string properties don't exceed the length of the provisioned columns. + * Values will be trimmed to the max length. + */ + private K validateFields(@NotNull AuditTypeProvider provider, @NotNull K type) + { + ObjectFactory factory = ObjectFactory.Registry.getFactory((Class)type.getClass()); + Map values = new CaseInsensitiveHashMap<>(); + factory.toMap(type, values); + + boolean changed = false; + Domain domain = provider.getDomain(); + + DomainKind domainKind = domain.getDomainKind(); + for (PropertyStorageSpec prop : domainKind.getBaseProperties(domain)) + { + Object value = values.get(prop.getName()); + if (prop.getJdbcType().isText() && value instanceof String s) + { + int scale = prop.getSize(); + if (s.length() > scale) + { + _log.warn("Audit field input : \n" + prop.getName() + "\nexceeded the maximum length : " + scale); + String trimmed = s.substring(0, scale-3) + "..."; + values.put(prop.getName(), trimmed); + changed = true; + } + } + } + + for (DomainProperty dp : domain.getProperties()) + { + // For now, only check for string length like we were doing for the old audit event fields + PropertyDescriptor pd = dp.getPropertyDescriptor(); + Object value = values.get(dp.getName()); + if (pd.isStringType() && value instanceof String s) + { + int scale = dp.getScale(); + if (scale > 0 && s.length() > scale) + { + _log.warn("Audit field input : \n" + pd.getName() + "\nexceeded the maximum length : " + scale); + String trimmed; + if (scale > 100) + trimmed = s.substring(0, scale-3) + "..."; + else + trimmed = s.substring(0, scale); + values.put(pd.getName(), trimmed); + changed = true; + } + } + } + + if (changed) + return factory.fromMap(values); + else + return type; + } +} From a1e7461b19e675786d865ab2eda1454d5be19b1a Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Mon, 6 Oct 2025 12:11:46 -0700 Subject: [PATCH 2/3] CRLF --- api/src/org/labkey/api/ApiModule.java | 2 - api/src/org/labkey/api/data/SQLFragment.java | 2719 ++++++------ .../org/labkey/api/data/StatementUtils.java | 3832 ++++++++--------- audit/src/org/labkey/audit/AuditLogImpl.java | 632 +-- .../org/labkey/audit/model/LogManager.java | 578 +-- 5 files changed, 3874 insertions(+), 3889 deletions(-) diff --git a/api/src/org/labkey/api/ApiModule.java b/api/src/org/labkey/api/ApiModule.java index b28cf36e71d..a14f28c3fef 100644 --- a/api/src/org/labkey/api/ApiModule.java +++ b/api/src/org/labkey/api/ApiModule.java @@ -504,13 +504,11 @@ public void registerServlets(ServletContext servletCtx) ParameterSubstitutionTest.class, Portal.TestCase.class, PropertyManager.TestCase.class, - //RateLimiter.TestCase.class, RecordFactory.TestCase.class, ResultSetDataIterator.TestCase.class, ResultSetSelectorTestCase.class, RoleSet.TestCase.class, RowTrackingResultSetWrapper.TestCase.class, - SQLFragment.IntegrationTestCase.class, SecurityManager.TestCase.class, SimpleTranslator.TranslateTestCase.class, SqlSelectorTestCase.class, diff --git a/api/src/org/labkey/api/data/SQLFragment.java b/api/src/org/labkey/api/data/SQLFragment.java index 9e8fffe00ac..be99434541e 100644 --- a/api/src/org/labkey/api/data/SQLFragment.java +++ b/api/src/org/labkey/api/data/SQLFragment.java @@ -1,1365 +1,1354 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.data; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.query.AliasManager; -import org.labkey.api.query.FieldKey; -import org.labkey.api.settings.AppProps; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JdbcUtil; -import org.labkey.api.util.Pair; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.TreeSet; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; - -import static org.labkey.api.query.ExprColumn.STR_TABLE_ALIAS; - -/** - * Holds both the SQL text and JDBC parameter values to use during invocation. - */ -public class SQLFragment implements Appendable, CharSequence -{ - public static final String FEATUREFLAG_DISABLE_STRICT_CHECKS = "sqlfragment-disable-strict-checks"; - - private String sql; - private StringBuilder sb = null; - private List params; // TODO: Should be List - - private final List tempTokens = new ArrayList<>(); // Hold refs to ensure they're not GC'd - - // use ordered map to make sql generation more deterministic (see collectCommonTableExpressions()) - private LinkedHashMap commonTableExpressionsMap = null; - - private static class CTE - { - CTE(@NotNull SqlDialect dialect, @NotNull String name) - { - this.dialect = dialect; - this.preferredName = name; - tokens.add("/*$*/" + GUID.makeGUID() + ":" + name + "/*$*/"); - } - - CTE(@NotNull SqlDialect dialect, @NotNull String name, SQLFragment sqlf, boolean recursive) - { - this(dialect, name); - this.sqlf = sqlf; - this.recursive = recursive; - } - - CTE(CTE from) - { - this.dialect = from.dialect; - this.preferredName = from.preferredName; - this.tokens.addAll(from.tokens); - this.sqlf = from.sqlf; - this.recursive = from.recursive; - } - - public CTE copy(boolean deep) - { - CTE copy = new CTE(this); - if (deep) - copy.sqlf = new SQLFragment().append(copy.sqlf); - return copy; - } - - private String token() - { - return tokens.iterator().next(); - } - - private final @NotNull SqlDialect dialect; - final String preferredName; - boolean recursive = false; // NOTE this is dialect dependant (getSql() does not take a dialect) - final Set tokens = new TreeSet<>(); - SQLFragment sqlf = null; - } - - public SQLFragment() - { - sql = ""; - } - - public SQLFragment(CharSequence charseq, @Nullable List params) - { - if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || - (StringUtils.countMatches(charseq, '\"') % 2) != 0 || - StringUtils.contains(charseq, ';')) - { - if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); - } - - // allow statement separators - this.sql = charseq.toString(); - if (null != params) - this.params = new ArrayList<>(params); - } - - - public SQLFragment(CharSequence sql, Object... params) - { - this(sql, Arrays.asList(params)); - } - - - public SQLFragment(SQLFragment other) - { - this(other,false); - } - - - public SQLFragment(SQLFragment other, boolean deep) - { - sql = other.getSqlCharSequence().toString(); - if (null != other.params) - addAll(other.params); - if (null != other.commonTableExpressionsMap && !other.commonTableExpressionsMap.isEmpty()) - { - if (null == this.commonTableExpressionsMap) - this.commonTableExpressionsMap = new LinkedHashMap<>(); - for (Map.Entry e : other.commonTableExpressionsMap.entrySet()) - { - CTE cte = e.getValue().copy(deep); - this.commonTableExpressionsMap.put(e.getKey(),cte); - } - } - this.tempTokens.addAll(other.tempTokens); - } - - - @Override - public boolean isEmpty() - { - return (null == sb || sb.isEmpty()) && (sql == null || sql.isEmpty()); - } - - - /* same as getSQL() but without CTE handling */ - public String getRawSQL() - { - return null != sb ? sb.toString() : null != sql ? sql : ""; - } - - /* - * Directly set the current SQL. - * - * This is useful for wrapping existing SQL, for instance adding a cast - * Obviously parameter number and order must remain unchanged - * - * This can also be used for processing sql scripts (e.g. module .sql update scripts) - */ - public SQLFragment setSqlUnsafe(String unsafe) - { - this.sql = unsafe; - this.sb = null; - return this; - } - - public static SQLFragment unsafe(String unsafe) - { - return new SQLFragment().setSqlUnsafe(unsafe); - } - - - private String replaceCteTokens(String self, String select, List> ctes) - { - for (Pair pair : ctes) - { - String alias = pair.first; - CTE cte = pair.second; - for (String token : cte.tokens) - { - select = Strings.CS.replace(select, token, alias); - } - } - if (null != self) - select = Strings.CS.replace(select, "$SELF$", self); - return select; - } - - - private List collectCommonTableExpressions() - { - List list = new ArrayList<>(); - _collectCommonTableExpressions(list); - return list; - } - - private void _collectCommonTableExpressions(List list) - { - if (null != commonTableExpressionsMap) - { - commonTableExpressionsMap.values().forEach(cte -> cte.sqlf._collectCommonTableExpressions(list)); - list.addAll(commonTableExpressionsMap.values()); - } - } - - - public String getSQL() - { - if (null == commonTableExpressionsMap || commonTableExpressionsMap.isEmpty()) - return null != sb ? sb.toString() : null != sql ? sql : ""; - - List commonTableExpressions = collectCommonTableExpressions(); - assert !commonTableExpressions.isEmpty(); - - boolean recursive = commonTableExpressions.stream() - .anyMatch(cte -> cte.recursive); - StringBuilder ret = new StringBuilder("WITH" + (recursive ? " RECURSIVE" : "")); - - // generate final aliases for each CTE */ - SqlDialect dialect = Objects.requireNonNull(commonTableExpressions.get(0).dialect); - AliasManager am = new AliasManager(dialect); - List> ctes = commonTableExpressions.stream() - .map(cte -> new Pair<>(am.decideAlias(cte.preferredName),cte)) - .collect(Collectors.toList()); - - String comma = "\n/*CTE*/\n\t"; - for (Pair p : ctes) - { - String alias = p.first; - CTE cte = p.second; - SQLFragment expr = cte.sqlf; - String sql = expr._getOwnSql(alias, ctes); - ret.append(comma).append(alias).append(" AS (").append(sql).append(")"); - comma = "\n,/*CTE*/\n\t"; - } - ret.append("\n"); - - String select = _getOwnSql( null, ctes ); - ret.append(replaceCteTokens(null, select, ctes)); - return ret.toString(); - } - - - private String _getOwnSql(String alias, List> ctes) - { - String ownSql = null != sb ? sb.toString() : null != this.sql ? this.sql : ""; - return replaceCteTokens(alias, ownSql, ctes); - } - - - static Pattern markerPattern = Pattern.compile("/\\*\\$\\*/.*/\\*\\$\\*/"); - - /* This is not an exhaustive .equals() test, but it give pretty good confidence that these statements are the same */ - static boolean debugCompareSQL(SQLFragment sql1, SQLFragment sql2) - { - String select1 = sql1.getRawSQL(); - String select2 = sql2.getRawSQL(); - - if ((null == sql1.commonTableExpressionsMap || sql1.commonTableExpressionsMap.isEmpty()) && - (null == sql2.commonTableExpressionsMap || sql2.commonTableExpressionsMap.isEmpty())) - return select1.equals(select2); - - select1 = markerPattern.matcher(select1).replaceAll("CTE"); - select2 = markerPattern.matcher(select2).replaceAll("CTE"); - if (!select1.equals(select2)) - return false; - - Set ctes1 = sql1.commonTableExpressionsMap.values().stream() - .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) - .collect(Collectors.toSet()); - Set ctes2 = sql2.commonTableExpressionsMap.values().stream() - .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) - .collect(Collectors.toSet()); - return ctes1.equals(ctes2); - } - - - // It is a little confusing that getString() does not return the same charsequence that this object purports to - // represent. However, this is a good "display value" for this object. - // see getSqlCharSequence() - @NotNull - public String toString() - { - return "SQLFragment@" + System.identityHashCode(this) + "\n" + JdbcUtil.format(this); - } - - - public String toDebugString() - { - return JdbcUtil.format(this); - } - - - public List getParams() - { - var ctes = collectCommonTableExpressions(); - List ret = new ArrayList<>(); - - for (var cte : ctes) - ret.addAll(cte.sqlf.getParamsNoCTEs()); - ret.addAll(getParamsNoCTEs()); - return Collections.unmodifiableList(ret); - } - - - public List> getParamsWithFragments() - { - var ctes = collectCommonTableExpressions(); - List> ret = new ArrayList<>(); - - for (CTE cte : ctes) - { - if (null != cte.sqlf && null != cte.sqlf.params) - { - for (int i = 0; i < cte.sqlf.params.size(); i++) - { - ret.add(new Pair<>(cte.sqlf, i)); - } - } - } - - if (null != params) - { - for (int i = 0; i < params.size(); i++) - { - ret.add(new Pair<>(this, i)); - } - } - return ret; - } - - private final static Object[] EMPTY_ARRAY = new Object[0]; - - public Object[] getParamsArray() - { - return null == params ? EMPTY_ARRAY : params.toArray(); - } - - public List getParamsNoCTEs() - { - return params == null ? Collections.emptyList() : Collections.unmodifiableList(params); - } - - private List getMutableParams() - { - if (!(params instanceof ArrayList)) - { - List t = new ArrayList<>(); - if (params != null) - t.addAll(params); - params = t; - } - return params; - } - - - private StringBuilder getStringBuilder() - { - if (null == sb) - sb = new StringBuilder(null==sql?"":sql); - return sb; - } - - - @Override - public SQLFragment append(CharSequence charseq) - { - if (null == charseq) - return this; - - if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || - (StringUtils.countMatches(charseq, '\"') % 2) != 0 || - StringUtils.contains(charseq, ';')) - { - if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); - } - - getStringBuilder().append(charseq); - return this; - } - - public SQLFragment appendIdentifier(DatabaseIdentifier id) - { - return append(id.getSql()); - } - - /** Functionally the same as append(CharSequence). This method just has different asserts */ - public SQLFragment appendIdentifier(CharSequence charseq) - { - if (null == charseq) - return this; - if (charseq instanceof SQLFragment sqlf) - { - if (0 != sqlf.getParamsArray().length) - throw new IllegalStateException("Unexpected SQL in appendIdentifier()"); - charseq = sqlf.getRawSQL(); - } - - String identifier = charseq.toString().strip(); - - if (STR_TABLE_ALIAS.equals(identifier)) - { - getStringBuilder().append(identifier); - return this; - } - - boolean malformed; - if (identifier.length() >= 2 && identifier.startsWith("\"") && identifier.endsWith("\"")) - malformed = (StringUtils.countMatches(identifier, '\"') % 2) != 0; - else if (identifier.length() >= 2 && identifier.startsWith("`") && identifier.endsWith("`")) - malformed = (StringUtils.countMatches(identifier, '`') % 2) != 0; - else - malformed = StringUtils.containsAny(identifier, "*/\\'\"`?;- \t\n"); - if (malformed && !AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - throw new IllegalArgumentException("SQLFragment.appendIdentifier(String) value appears to be incorrectly formatted: " + identifier); - - getStringBuilder().append(charseq); - return this; - } - - // just to save some typing - public SQLFragment appendDottedIdentifiers(CharSequence table, DatabaseIdentifier col) - { - return appendIdentifier(table).append(".").appendIdentifier(col); - } - - // just to save some typing - public SQLFragment appendDottedIdentifiers(CharSequence... ids) - { - var dot = ""; - for (var id : ids) - { - append(dot).appendIdentifier(id); - dot = "."; - } - return this; - } - - /** append End Of Statement */ - public SQLFragment appendEOS() - { - getStringBuilder().append(";\n"); - return this; - } - - - @Override - public SQLFragment append(CharSequence csq, int start, int end) - { - append(csq.subSequence(start, end)); - return this; - } - - /** Adds the container's ID as an in-line string constant to the SQL */ - public SQLFragment appendValue(Container c) - { - if (null == c) - return appendNull(); - return appendValue(c, null); - } - - public SQLFragment appendValue(@NotNull Container c, SqlDialect dialect) - { - appendValue(c.getEntityId(), dialect); - String name = c.getName(); - if (!StringUtils.containsAny(name,"*/\\'\"?")) - append("/* ").append(name).append(" */"); - return this; - } - - public SQLFragment appendNull() - { - getStringBuilder().append("NULL"); - return this; - } - - public SQLFragment appendValue(Boolean B, @NotNull SqlDialect dialect) - { - if (null == B) - return append("CAST(NULL AS ").append(dialect.getBooleanDataType()).append(")"); - getStringBuilder().append(B ? dialect.getBooleanTRUE() : dialect.getBooleanFALSE()); - return this; - } - - public SQLFragment appendValue(Integer I) - { - if (null == I) - return appendNull(); - getStringBuilder().append(I.intValue()); - return this; - } - - public SQLFragment appendValue(int i) - { - getStringBuilder().append(i); - return this; - } - - - public SQLFragment appendValue(Long L) - { - if (null == L) - return appendNull(); - getStringBuilder().append((long)L); - return this; - } - - public SQLFragment appendValue(long l) - { - getStringBuilder().append(l); - return this; - } - - public SQLFragment appendValue(Float F) - { - if (null == F) - return appendNull(); - return appendValue(F.floatValue()); - } - - public SQLFragment appendValue(float f) - { - if (Float.isFinite(f)) - { - getStringBuilder().append(f); - } - else - { - getStringBuilder().append("?"); - add(f); - } - return this; - } - - public SQLFragment appendValue(Double D) - { - if (null == D) - return appendNull(); - else - return appendValue(D.doubleValue()); - } - - public SQLFragment appendValue(double d) - { - if (Double.isFinite(d)) - { - getStringBuilder().append(d); - } - else - { - getStringBuilder().append("?"); - add(d); - } - return this; - } - - public SQLFragment appendValue(Number N) - { - if (null == N) - return appendNull(); - - if (N instanceof Quantity q) - N = q.value(); - - if (N instanceof BigDecimal || N instanceof BigInteger || N instanceof Long) - { - getStringBuilder().append(N); - } - else if (Double.isFinite(N.doubleValue())) - { - getStringBuilder().append(N); - } - else - { - getStringBuilder().append(" ? "); - add(N); - } - return this; - } - - // Issue 27534: Stop using {fn now()} in function declarations - // Issue 48864: Query Table's use of web server time can cause discrepancies in created/modified timestamps - public final SQLFragment appendValue(NowTimestamp now) - { - if (null == now) - return appendNull(); - getStringBuilder().append("CURRENT_TIMESTAMP"); - return this; - } - - public final SQLFragment appendValue(java.util.Date d) - { - if (null == d) - return appendNull(); - if (d.getClass() == java.util.Date.class) - getStringBuilder().append("{ts '").append(new Timestamp(d.getTime())).append("'}"); - else if (d.getClass() == java.sql.Timestamp.class) - getStringBuilder().append("{ts '").append(d).append("'}"); - else if (d.getClass() == java.sql.Date.class) - getStringBuilder().append("{d '").append(d).append("'}"); - else - throw new IllegalStateException("Unexpected date type: " + d.getClass().getName()); - return this; - } - - public SQLFragment appendValue(GUID g) - { - return appendValue(g, null); - } - - public SQLFragment appendValue(GUID g, SqlDialect d) - { - if (null == g) - return appendNull(); - // doesn't need StringHandler, just hex and hyphen - String sqlGUID = "'" + g + "'"; - // I'm testing dialect type, because some dialects do not support getGuidType(), and postgers uses VARCHAR anyway - if (null != d && d.isSqlServer()) - getStringBuilder().append("CAST(").append(sqlGUID).append(" AS UNIQUEIDENTIFIER)"); - else - getStringBuilder().append(sqlGUID); - return this; - } - - public SQLFragment appendValue(Enum e) - { - if (null == e) - return appendNull(); - String name = e.name(); - // Enum.name() usually returns a simple string (a legal java identifier), this is a paranoia check. - if (name.contains("'")) - throw new IllegalStateException(); - getStringBuilder().append("'").append(name).append("'"); - return this; - } - - public SQLFragment append(FieldKey fk) - { - if (null == fk) - return appendNull(); - append(String.valueOf(fk)); - return this; - } - - - /** Adds the object as a JDBC parameter value */ - public SQLFragment add(Object p) - { - getMutableParams().add(p); - return this; - } - - - /** Adds the objects as JDBC parameter values */ - public SQLFragment addAll(Collection l) - { - getMutableParams().addAll(l); - return this; - } - - - /** Adds the objects as JDBC parameter values */ - public SQLFragment addAll(Object... values) - { - if (values == null) - return this; - addAll(Arrays.asList(values)); - return this; - } - - - /** Sets the parameter at the index to the object's value */ - public void set(int i, Object p) - { - getMutableParams().set(i,p); - } - - /** Append both the SQL and the parameters from the other SQLFragment to this SQLFragment */ - public SQLFragment append(SQLFragment f) - { - if (null != f.sb) - getStringBuilder().append(f.sb); - else - getStringBuilder().append(f.sql); - if (null != f.params) - addAll(f.params); - mergeCommonTableExpressions(f); - tempTokens.addAll(f.tempTokens); - return this; - } - - public SQLFragment append(@NotNull Iterable fragments, @NotNull String separator) - { - String s = ""; - for (SQLFragment fragment : fragments) - { - append(s); - s = separator; - append(fragment); - } - return this; - } - - // return boolean so this can be used in an assert. passing in a dialect is not ideal, but parsing comments out - // before submitting the fragment is not reliable and holding statements & comments separately (to eliminate the - // need to parse them) isn't particularly easy... so punt for now. - public boolean appendComment(String comment, SqlDialect dialect) - { - if (dialect.supportsComments()) - { - StringBuilder sb = getStringBuilder(); - int len = sb.length(); - if (len > 0 && sb.charAt(len-1) != '\n') - sb.append('\n'); - sb.append("\n-- "); - boolean truncated = comment.length() > 1000; - if (truncated) - comment = comment.substring(0,1000); - sb.append(comment); - if (StringUtils.countMatches(comment, "'")%2==1) - sb.append("'"); - if (truncated) - sb.append("..."); - sb.append('\n'); - } - return true; - } - - - /** see also append(TableInfo, String alias) */ - public SQLFragment append(TableInfo table) - { - SQLFragment s = table.getSQLName(); - if (s != null) - return append(s); - - String alias = table.getSqlDialect().makeLegalIdentifier(table.getName()); - return append(table.getFromSQL(alias)); - } - - /** Add a table/query to the SQL with an alias, as used in a FROM clause */ - public SQLFragment append(TableInfo table, String alias) - { - return append(table.getFromSQL(alias)); - } - - /** Add to the SQL */ - @Override - public SQLFragment append(char ch) - { - getStringBuilder().append(ch); - return this; - } - - /** This is like appendValue(CharSequence s), but force use of literal syntax - * CAUTIONARY NOTE: String literals in PostgresSQL are tricky because of overloaded functions - * array_agg('string') fails array_agg('string'::VARCHAR) works - * json_object('{}) works json_object('string'::VARCHAR) fails - * In the case of json_object() it expects TEXT. Postgres will promote 'json' to TEXT, but not 'json'::VARCHAR - */ - public SQLFragment appendStringLiteral(CharSequence s, @NotNull SqlDialect d) - { - if (null==s) - return appendNull(); - getStringBuilder().append(d.getStringHandler().quoteStringLiteral(s.toString())); - return this; - } - - /** Add to the SQL as either an in-line string literal or as a JDBC parameter depending on whether it would need escaping */ - public SQLFragment appendValue(CharSequence s) - { - return appendValue(s, null); - } - - public SQLFragment appendValue(CharSequence s, SqlDialect d) - { - if (null==s) - return appendNull(); - if (null==d || s.length() > 200) - return append("?").add(s.toString()); - appendStringLiteral(s, d); - return this; - } - - public SQLFragment appendInClause(@NotNull Collection params, SqlDialect dialect) - { - dialect.appendInClauseSql(this, params); - return this; - } - - public CharSequence getSqlCharSequence() - { - if (null != sb) - { - return sb; - } - return sql; - } - - public void insert(int index, SQLFragment sql) - { - if (!sql.getParams().isEmpty()) - { - throw new IllegalArgumentException("Not supported for SQLFragments with parameters - they must be inserted/merged separately"); - } - if (sql.commonTableExpressionsMap != null && !sql.commonTableExpressionsMap.isEmpty()) - { - throw new IllegalArgumentException("Not supported for SQLFragments with CTEs - they must be inserted/merged separately"); - } - if (!tempTokens.isEmpty()) - { - throw new IllegalArgumentException("Not supported for SQLFragments with temp tokens - they must be inserted/merged separately"); - } - getStringBuilder().insert(index, sql.getRawSQL()); - } - - /** Insert into the SQL */ - public void insert(int index, String str) - { - if ((StringUtils.countMatches(str, '\'') % 2) != 0 || - (StringUtils.countMatches(str, '\"') % 2) != 0 || - StringUtils.contains(str, ';')) - { - if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - throw new IllegalArgumentException("SQLFragment.insert(int,String) does not allow semicolons or unmatched quotes"); - } - - getStringBuilder().insert(index, str); - } - - /** Insert this SQLFragment's SQL and parameters at the start of the existing SQL and parameters */ - public void prepend(SQLFragment sql) - { - getStringBuilder().insert(0, sql.getSqlCharSequence().toString()); - if (null != sql.params) - getMutableParams().addAll(0, sql.params); - mergeCommonTableExpressions(sql); - } - - - public int indexOf(String str) - { - return getStringBuilder().indexOf(str); - } - - - // Display query in "English" (display SQL with params substituted) - // with a little more work could probably be made to be SQL legal - public String getFilterText() - { - String sql = getSQL().replaceFirst("WHERE ", ""); - List params = getParams(); - for (Object param1 : params) - { - String param = param1.toString(); - param = param.replaceAll("\\\\", "\\\\\\\\"); - param = param.replaceAll("\\$", "\\\\\\$"); - sql = sql.replaceFirst("\\?", param); - } - return sql.replaceAll("\"", ""); - } - - - @Override - public char charAt(int index) - { - return getSqlCharSequence().charAt(index); - } - - @Override - public int length() - { - return getSqlCharSequence().length(); - } - - @Override - public @NotNull CharSequence subSequence(int start, int end) - { - return getSqlCharSequence().subSequence(start, end); - } - - /** - * KEY is used as a faster way to look for equivalent CTE expressions. - * returning a name here allows us to potentially merge CTE at add time - * - * if you don't have a key you can just use sqlf.toString() - */ - public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf) - { - return addCommonTableExpression(dialect, key, proposedName, sqlf, false); - } - - public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf, boolean recursive) - { - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - CTE prev = commonTableExpressionsMap.get(key); - if (null != prev) - return prev.token(); - CTE cte = new CTE(dialect, proposedName, sqlf, recursive); - commonTableExpressionsMap.put(key, cte); - return cte.token(); - } - - public String createCommonTableExpressionToken(SqlDialect dialect, Object key, String proposedName) - { - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - CTE prev = commonTableExpressionsMap.get(key); - if (null != prev) - throw new IllegalStateException("Cannot create CTE token from already used key."); - CTE cte = new CTE(dialect ,proposedName); - commonTableExpressionsMap.put(key, cte); - return cte.token(); - } - - public void setCommonTableExpressionSql(Object key, SQLFragment sqlf, boolean recursive) - { - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - - if (null != sqlf.commonTableExpressionsMap && !sqlf.commonTableExpressionsMap.isEmpty()) - { - // Need to merge CTEs up; this.cte depends on newSql.ctes, so they need to come first - SQLFragment newSql = new SQLFragment(sqlf); - LinkedHashMap toMap = new LinkedHashMap<>(newSql.commonTableExpressionsMap); - for (Map.Entry e : commonTableExpressionsMap.entrySet()) - { - CTE from = e.getValue(); - CTE to = toMap.get(e.getKey()); - if (null != to) - to.tokens.addAll(from.tokens); - else - toMap.put(e.getKey(), from.copy(false)); - } - - commonTableExpressionsMap = toMap; - newSql.commonTableExpressionsMap = null; - sqlf = newSql; - } - - CTE cte = commonTableExpressionsMap.get(key); - if (null == cte) - throw new IllegalStateException("CTE not found."); - cte.sqlf = sqlf; - cte.recursive = recursive; - } - - - private void mergeCommonTableExpressions(SQLFragment sqlFrom) - { - if (null == sqlFrom.commonTableExpressionsMap || sqlFrom.commonTableExpressionsMap.isEmpty()) - return; - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - for (Map.Entry e : sqlFrom.commonTableExpressionsMap.entrySet()) - { - CTE from = e.getValue(); - CTE to = commonTableExpressionsMap.get(e.getKey()); - if (null != to) - to.tokens.addAll(from.tokens); - else - commonTableExpressionsMap.put(e.getKey(), from.copy(false)); - } - } - - - public void addTempToken(Object tempToken) - { - tempTokens.add(tempToken); - } - - public void addTempTokens(SQLFragment other) - { - tempTokens.add(other.tempTokens); - } - - public static SQLFragment prettyPrint(SQLFragment from) - { - SQLFragment sqlf = new SQLFragment(from); - - String s = from.getSqlCharSequence().toString(); - StringBuilder sb = new StringBuilder(s.length() + 200); - String[] lines = StringUtils.split(s, '\n'); - int indent = 0; - - for (String line : lines) - { - String t = line.trim(); - - if (t.isEmpty()) - continue; - - if (t.startsWith("-- params = b.getParams(); - assertEquals(2,params.size()); - assertEquals(5, params.get(0)); - assertEquals("xxyzzy", params.get(1)); - - - SQLFragment c = new SQLFragment(b); - assertEquals(""" - WITH - /*CTE*/ - \tCTE AS (SELECT a FROM b WHERE x=?) - SELECT * FROM CTE WHERE y=?""", - c.getSQL()); - assertEquals(""" - WITH - /*CTE*/ - \tCTE AS (SELECT a FROM b WHERE x=5) - SELECT * FROM CTE WHERE y='xxyzzy'""", - filterDebugString(c.toDebugString())); - params = c.getParams(); - assertEquals(2,params.size()); - assertEquals(5, params.get(0)); - assertEquals("xxyzzy", params.get(1)); - - - // combining - - SQLFragment sqlf = new SQLFragment(); - String token = sqlf.addCommonTableExpression(dialect, "KEY_A", "cte1", new SQLFragment("SELECT * FROM a")); - sqlf.append("SELECT * FROM ").append(token).append(" _1"); - - assertEquals(""" - WITH - /*CTE*/ - \tcte1 AS (SELECT * FROM a) - SELECT * FROM cte1 _1""", - sqlf.getSQL()); - - SQLFragment sqlf2 = new SQLFragment(); - String token2 = sqlf2.addCommonTableExpression(dialect, "KEY_A", "cte2", new SQLFragment("SELECT * FROM a")); - sqlf2.append("SELECT * FROM ").append(token2).append(" _2"); - assertEquals(""" - WITH - /*CTE*/ - \tcte2 AS (SELECT * FROM a) - SELECT * FROM cte2 _2""", - sqlf2.getSQL()); - - SQLFragment sqlf3 = new SQLFragment(); - String token3 = sqlf3.addCommonTableExpression(dialect, "KEY_B", "cte3", new SQLFragment("SELECT * FROM b")); - sqlf3.append("SELECT * FROM ").append(token3).append(" _3"); - assertEquals(""" - WITH - /*CTE*/ - \tcte3 AS (SELECT * FROM b) - SELECT * FROM cte3 _3""", - sqlf3.getSQL()); - - SQLFragment union = new SQLFragment(); - union.append(sqlf); - union.append("\nUNION\n"); - union.append(sqlf2); - union.append("\nUNION\n"); - union.append(sqlf3); - assertEquals(""" - WITH - /*CTE*/ - \tcte1 AS (SELECT * FROM a) - ,/*CTE*/ - \tcte3 AS (SELECT * FROM b) - SELECT * FROM cte1 _1 - UNION - SELECT * FROM cte1 _2 - UNION - SELECT * FROM cte3 _3""", - union.getSQL()); - } - - @Test - public void nested_cte() - { - // one-level cte using cteToken (CTE fragment 'a' does not contain a CTE) - { - SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); - assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); - SQLFragment b = new SQLFragment(); - String cteToken = b.addCommonTableExpression(dialect, new Object(), "CTE", a); - b.append("SELECT * FROM ").append(cteToken).append(" WHERE p=?").add("parameterTWO"); - assertEquals(""" - WITH - /*CTE*/ - \tCTE AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) - SELECT * FROM CTE WHERE p='parameterTWO'""", - filterDebugString(b.toDebugString())); - assertEquals("parameterONE", b.getParams().get(0)); - } - - // two-level cte using cteTokens (CTE fragment 'b' contains a CTE of fragment a) - { - SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); - assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); - SQLFragment b = new SQLFragment(); - String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); - b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterTWO"); - SQLFragment c = new SQLFragment(); - String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); - c.append("SELECT * FROM ").append(cteTokenB).append(" WHERE i=?").add(3); - assertEquals(""" - WITH - /*CTE*/ - \tA_ AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) - ,/*CTE*/ - \tB_ AS (SELECT * FROM A_ WHERE p='parameterTWO') - SELECT * FROM B_ WHERE i=3""", - filterDebugString(c.toDebugString())); - List params = c.getParams(); - assertEquals(3, params.size()); - assertEquals("parameterONE", params.get(0)); - assertEquals("parameterTWO", params.get(1)); - assertEquals(3, params.get(2)); - } - - // Same as previous but top-level query has both a nested and non-nested CTE - { - SQLFragment a = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); - SQLFragment a2 = new SQLFragment("SELECT 2 as i, 'Atwo' as s, CAST(? AS VARCHAR) as p", "parameterAtwo"); - SQLFragment b = new SQLFragment(); - String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); - b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); - SQLFragment c = new SQLFragment(); - String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); - String cteTokenA2 = c.addCommonTableExpression(dialect, new Object(), "A2_", a2); - c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); - assertEquals(""" - WITH - /*CTE*/ - \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) - ,/*CTE*/ - \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') - ,/*CTE*/ - \tA2_ AS (SELECT 2 as i, 'Atwo' as s, CAST('parameterAtwo' AS VARCHAR) as p) - SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", - filterDebugString(c.toDebugString())); - List params = c.getParams(); - assertEquals(4, params.size()); - assertEquals("parameterAone", params.get(0)); - assertEquals("parameterB", params.get(1)); - assertEquals("parameterAtwo", params.get(2)); - assertEquals(4, params.get(3)); - } - - // Same as previous but two of the CTEs are the same and should be collapsed (e.g. imagine a container filter implemented with a CTE) - // TODO, we only collapse CTEs that are siblings - { - SQLFragment cf = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); - SQLFragment b = new SQLFragment(); - String cteTokenA = b.addCommonTableExpression(dialect, "CTE_KEY_CF", "A_", cf); - b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); - SQLFragment c = new SQLFragment(); - String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); - String cteTokenA2 = c.addCommonTableExpression(dialect, "CTE_KEY_CF", "A2_", cf); - c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); - assertEquals(""" - WITH - /*CTE*/ - \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) - ,/*CTE*/ - \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') - ,/*CTE*/ - \tA2_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) - SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", - filterDebugString(c.toDebugString())); - List params = c.getParams(); - assertEquals(4, params.size()); - assertEquals("parameterAone", params.get(0)); - assertEquals("parameterB", params.get(1)); - assertEquals("parameterAone", params.get(2)); - assertEquals(4, params.get(3)); - } - } - - - private void shouldFail(Runnable r) - { - try - { - r.run(); - if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - fail("Expected IllegalArgumentException"); - } - catch (IllegalArgumentException e) - { - if (AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - fail("Did not expect IllegalArgumentException"); - } - } - - - @Test - public void testIllegalArgument() - { - shouldFail(() -> new SQLFragment(";")); - shouldFail(() -> new SQLFragment().append(";")); - shouldFail(() -> new SQLFragment("AND name='")); - shouldFail(() -> new SQLFragment().append("AND name = '")); - shouldFail(() -> new SQLFragment().append("AND name = 'Robert'); DROP TABLE Students; --")); - - shouldFail(() -> new SQLFragment().appendIdentifier("column name")); - shouldFail(() -> new SQLFragment().appendIdentifier("?")); - shouldFail(() -> new SQLFragment().appendIdentifier(";")); - shouldFail(() -> new SQLFragment().appendIdentifier("\"column\"name\"")); - } - - - String mysqlQuoteIdentifier(String id) - { - return "`" + id.replaceAll("`", "``") + "`"; - } - - @Test - public void testMysql() - { - // OK - new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("mysql")); - new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my`sql")); - new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my\"sql")); - - // not OK - shouldFail(() -> new SQLFragment().appendIdentifier("`")); - shouldFail(() -> new SQLFragment().appendIdentifier("`a`a`")); - } - } - - - public static class IntegrationTestCase extends Assert - { - @Test - public void test() - { - // try some Dialect stuff and CTE executed against core schema - } - } - - - @Override - public boolean equals(Object obj) - { - if (!(obj instanceof SQLFragment other)) - { - return false; - } - return getSQL().equals(other.getSQL()) && getParams().equals(other.getParams()); - } - - /** - * Joins the SQLFragments in the provided {@code Iterable} into a single SQLFragment. The SQL is joined by string - * concatenation using the provided separator. The parameters are combined to form the new parameter list. - * - * @param fragments SQLFragments to join together - * @param separator Separator to use on the SQL portion - * @return A new SQLFragment that joins all the SQLFragments - */ - public static SQLFragment join(Iterable fragments, String separator) - { - if (separator.contains("?")) - throw new IllegalStateException("separator must not include a parameter marker"); - - // Join all the SQL statements - String sql = StreamSupport.stream(fragments.spliterator(), false) - .map(SQLFragment::getSQL) - .collect(Collectors.joining(separator)); - - // Collect all the parameters to a single list - List params = StreamSupport.stream(fragments.spliterator(), false) - .map(SQLFragment::getParams) - .flatMap(Collection::stream) - .collect(Collectors.toList()); - - return new SQLFragment(sql, params); - } - - // Marker interface to hint that this value may be replaced by CURRENT_TIMESTAMP - public static class NowTimestamp extends java.sql.Timestamp - { - public NowTimestamp() - { - this(System.currentTimeMillis()); - } - - public NowTimestamp(long ms) - { - super(ms); - } - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.data; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.query.AliasManager; +import org.labkey.api.query.FieldKey; +import org.labkey.api.settings.AppProps; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JdbcUtil; +import org.labkey.api.util.Pair; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +import static org.labkey.api.query.ExprColumn.STR_TABLE_ALIAS; + +/** + * Holds both the SQL text and JDBC parameter values to use during invocation. + */ +public class SQLFragment implements Appendable, CharSequence +{ + public static final String FEATUREFLAG_DISABLE_STRICT_CHECKS = "sqlfragment-disable-strict-checks"; + + private String sql; + private StringBuilder sb = null; + private List params; // TODO: Should be List + + private final List tempTokens = new ArrayList<>(); // Hold refs to ensure they're not GC'd + + // use ordered map to make sql generation more deterministic (see collectCommonTableExpressions()) + private LinkedHashMap commonTableExpressionsMap = null; + + private static class CTE + { + CTE(@NotNull SqlDialect dialect, @NotNull String name) + { + this.dialect = dialect; + this.preferredName = name; + tokens.add("/*$*/" + GUID.makeGUID() + ":" + name + "/*$*/"); + } + + CTE(@NotNull SqlDialect dialect, @NotNull String name, SQLFragment sqlf, boolean recursive) + { + this(dialect, name); + this.sqlf = sqlf; + this.recursive = recursive; + } + + CTE(CTE from) + { + this.dialect = from.dialect; + this.preferredName = from.preferredName; + this.tokens.addAll(from.tokens); + this.sqlf = from.sqlf; + this.recursive = from.recursive; + } + + public CTE copy(boolean deep) + { + CTE copy = new CTE(this); + if (deep) + copy.sqlf = new SQLFragment().append(copy.sqlf); + return copy; + } + + private String token() + { + return tokens.iterator().next(); + } + + private final @NotNull SqlDialect dialect; + final String preferredName; + boolean recursive = false; // NOTE this is dialect dependant (getSql() does not take a dialect) + final Set tokens = new TreeSet<>(); + SQLFragment sqlf = null; + } + + public SQLFragment() + { + sql = ""; + } + + public SQLFragment(CharSequence charseq, @Nullable List params) + { + if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || + (StringUtils.countMatches(charseq, '\"') % 2) != 0 || + StringUtils.contains(charseq, ';')) + { + if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); + } + + // allow statement separators + this.sql = charseq.toString(); + if (null != params) + this.params = new ArrayList<>(params); + } + + + public SQLFragment(CharSequence sql, Object... params) + { + this(sql, Arrays.asList(params)); + } + + + public SQLFragment(SQLFragment other) + { + this(other,false); + } + + + public SQLFragment(SQLFragment other, boolean deep) + { + sql = other.getSqlCharSequence().toString(); + if (null != other.params) + addAll(other.params); + if (null != other.commonTableExpressionsMap && !other.commonTableExpressionsMap.isEmpty()) + { + if (null == this.commonTableExpressionsMap) + this.commonTableExpressionsMap = new LinkedHashMap<>(); + for (Map.Entry e : other.commonTableExpressionsMap.entrySet()) + { + CTE cte = e.getValue().copy(deep); + this.commonTableExpressionsMap.put(e.getKey(),cte); + } + } + this.tempTokens.addAll(other.tempTokens); + } + + + @Override + public boolean isEmpty() + { + return (null == sb || sb.isEmpty()) && (sql == null || sql.isEmpty()); + } + + + /* same as getSQL() but without CTE handling */ + public String getRawSQL() + { + return null != sb ? sb.toString() : null != sql ? sql : ""; + } + + /* + * Directly set the current SQL. + * + * This is useful for wrapping existing SQL, for instance adding a cast + * Obviously parameter number and order must remain unchanged + * + * This can also be used for processing sql scripts (e.g. module .sql update scripts) + */ + public SQLFragment setSqlUnsafe(String unsafe) + { + this.sql = unsafe; + this.sb = null; + return this; + } + + public static SQLFragment unsafe(String unsafe) + { + return new SQLFragment().setSqlUnsafe(unsafe); + } + + + private String replaceCteTokens(String self, String select, List> ctes) + { + for (Pair pair : ctes) + { + String alias = pair.first; + CTE cte = pair.second; + for (String token : cte.tokens) + { + select = Strings.CS.replace(select, token, alias); + } + } + if (null != self) + select = Strings.CS.replace(select, "$SELF$", self); + return select; + } + + + private List collectCommonTableExpressions() + { + List list = new ArrayList<>(); + _collectCommonTableExpressions(list); + return list; + } + + private void _collectCommonTableExpressions(List list) + { + if (null != commonTableExpressionsMap) + { + commonTableExpressionsMap.values().forEach(cte -> cte.sqlf._collectCommonTableExpressions(list)); + list.addAll(commonTableExpressionsMap.values()); + } + } + + + public String getSQL() + { + if (null == commonTableExpressionsMap || commonTableExpressionsMap.isEmpty()) + return null != sb ? sb.toString() : null != sql ? sql : ""; + + List commonTableExpressions = collectCommonTableExpressions(); + assert !commonTableExpressions.isEmpty(); + + boolean recursive = commonTableExpressions.stream() + .anyMatch(cte -> cte.recursive); + StringBuilder ret = new StringBuilder("WITH" + (recursive ? " RECURSIVE" : "")); + + // generate final aliases for each CTE */ + SqlDialect dialect = Objects.requireNonNull(commonTableExpressions.get(0).dialect); + AliasManager am = new AliasManager(dialect); + List> ctes = commonTableExpressions.stream() + .map(cte -> new Pair<>(am.decideAlias(cte.preferredName),cte)) + .collect(Collectors.toList()); + + String comma = "\n/*CTE*/\n\t"; + for (Pair p : ctes) + { + String alias = p.first; + CTE cte = p.second; + SQLFragment expr = cte.sqlf; + String sql = expr._getOwnSql(alias, ctes); + ret.append(comma).append(alias).append(" AS (").append(sql).append(")"); + comma = "\n,/*CTE*/\n\t"; + } + ret.append("\n"); + + String select = _getOwnSql( null, ctes ); + ret.append(replaceCteTokens(null, select, ctes)); + return ret.toString(); + } + + + private String _getOwnSql(String alias, List> ctes) + { + String ownSql = null != sb ? sb.toString() : null != this.sql ? this.sql : ""; + return replaceCteTokens(alias, ownSql, ctes); + } + + + static Pattern markerPattern = Pattern.compile("/\\*\\$\\*/.*/\\*\\$\\*/"); + + /* This is not an exhaustive .equals() test, but it give pretty good confidence that these statements are the same */ + static boolean debugCompareSQL(SQLFragment sql1, SQLFragment sql2) + { + String select1 = sql1.getRawSQL(); + String select2 = sql2.getRawSQL(); + + if ((null == sql1.commonTableExpressionsMap || sql1.commonTableExpressionsMap.isEmpty()) && + (null == sql2.commonTableExpressionsMap || sql2.commonTableExpressionsMap.isEmpty())) + return select1.equals(select2); + + select1 = markerPattern.matcher(select1).replaceAll("CTE"); + select2 = markerPattern.matcher(select2).replaceAll("CTE"); + if (!select1.equals(select2)) + return false; + + Set ctes1 = sql1.commonTableExpressionsMap.values().stream() + .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) + .collect(Collectors.toSet()); + Set ctes2 = sql2.commonTableExpressionsMap.values().stream() + .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) + .collect(Collectors.toSet()); + return ctes1.equals(ctes2); + } + + + // It is a little confusing that getString() does not return the same charsequence that this object purports to + // represent. However, this is a good "display value" for this object. + // see getSqlCharSequence() + @NotNull + public String toString() + { + return "SQLFragment@" + System.identityHashCode(this) + "\n" + JdbcUtil.format(this); + } + + + public String toDebugString() + { + return JdbcUtil.format(this); + } + + + public List getParams() + { + var ctes = collectCommonTableExpressions(); + List ret = new ArrayList<>(); + + for (var cte : ctes) + ret.addAll(cte.sqlf.getParamsNoCTEs()); + ret.addAll(getParamsNoCTEs()); + return Collections.unmodifiableList(ret); + } + + + public List> getParamsWithFragments() + { + var ctes = collectCommonTableExpressions(); + List> ret = new ArrayList<>(); + + for (CTE cte : ctes) + { + if (null != cte.sqlf && null != cte.sqlf.params) + { + for (int i = 0; i < cte.sqlf.params.size(); i++) + { + ret.add(new Pair<>(cte.sqlf, i)); + } + } + } + + if (null != params) + { + for (int i = 0; i < params.size(); i++) + { + ret.add(new Pair<>(this, i)); + } + } + return ret; + } + + private final static Object[] EMPTY_ARRAY = new Object[0]; + + public Object[] getParamsArray() + { + return null == params ? EMPTY_ARRAY : params.toArray(); + } + + public List getParamsNoCTEs() + { + return params == null ? Collections.emptyList() : Collections.unmodifiableList(params); + } + + private List getMutableParams() + { + if (!(params instanceof ArrayList)) + { + List t = new ArrayList<>(); + if (params != null) + t.addAll(params); + params = t; + } + return params; + } + + + private StringBuilder getStringBuilder() + { + if (null == sb) + sb = new StringBuilder(null==sql?"":sql); + return sb; + } + + + @Override + public SQLFragment append(CharSequence charseq) + { + if (null == charseq) + return this; + + if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || + (StringUtils.countMatches(charseq, '\"') % 2) != 0 || + StringUtils.contains(charseq, ';')) + { + if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); + } + + getStringBuilder().append(charseq); + return this; + } + + public SQLFragment appendIdentifier(DatabaseIdentifier id) + { + return append(id.getSql()); + } + + /** Functionally the same as append(CharSequence). This method just has different asserts */ + public SQLFragment appendIdentifier(CharSequence charseq) + { + if (null == charseq) + return this; + if (charseq instanceof SQLFragment sqlf) + { + if (0 != sqlf.getParamsArray().length) + throw new IllegalStateException("Unexpected SQL in appendIdentifier()"); + charseq = sqlf.getRawSQL(); + } + + String identifier = charseq.toString().strip(); + + if (STR_TABLE_ALIAS.equals(identifier)) + { + getStringBuilder().append(identifier); + return this; + } + + boolean malformed; + if (identifier.length() >= 2 && identifier.startsWith("\"") && identifier.endsWith("\"")) + malformed = (StringUtils.countMatches(identifier, '\"') % 2) != 0; + else if (identifier.length() >= 2 && identifier.startsWith("`") && identifier.endsWith("`")) + malformed = (StringUtils.countMatches(identifier, '`') % 2) != 0; + else + malformed = StringUtils.containsAny(identifier, "*/\\'\"`?;- \t\n"); + if (malformed && !AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + throw new IllegalArgumentException("SQLFragment.appendIdentifier(String) value appears to be incorrectly formatted: " + identifier); + + getStringBuilder().append(charseq); + return this; + } + + // just to save some typing + public SQLFragment appendDottedIdentifiers(CharSequence table, DatabaseIdentifier col) + { + return appendIdentifier(table).append(".").appendIdentifier(col); + } + + // just to save some typing + public SQLFragment appendDottedIdentifiers(CharSequence... ids) + { + var dot = ""; + for (var id : ids) + { + append(dot).appendIdentifier(id); + dot = "."; + } + return this; + } + + /** append End Of Statement */ + public SQLFragment appendEOS() + { + getStringBuilder().append(";\n"); + return this; + } + + + @Override + public SQLFragment append(CharSequence csq, int start, int end) + { + append(csq.subSequence(start, end)); + return this; + } + + /** Adds the container's ID as an in-line string constant to the SQL */ + public SQLFragment appendValue(Container c) + { + if (null == c) + return appendNull(); + return appendValue(c, null); + } + + public SQLFragment appendValue(@NotNull Container c, SqlDialect dialect) + { + appendValue(c.getEntityId(), dialect); + String name = c.getName(); + if (!StringUtils.containsAny(name,"*/\\'\"?")) + append("/* ").append(name).append(" */"); + return this; + } + + public SQLFragment appendNull() + { + getStringBuilder().append("NULL"); + return this; + } + + public SQLFragment appendValue(Boolean B, @NotNull SqlDialect dialect) + { + if (null == B) + return append("CAST(NULL AS ").append(dialect.getBooleanDataType()).append(")"); + getStringBuilder().append(B ? dialect.getBooleanTRUE() : dialect.getBooleanFALSE()); + return this; + } + + public SQLFragment appendValue(Integer I) + { + if (null == I) + return appendNull(); + getStringBuilder().append(I.intValue()); + return this; + } + + public SQLFragment appendValue(int i) + { + getStringBuilder().append(i); + return this; + } + + + public SQLFragment appendValue(Long L) + { + if (null == L) + return appendNull(); + getStringBuilder().append((long)L); + return this; + } + + public SQLFragment appendValue(long l) + { + getStringBuilder().append(l); + return this; + } + + public SQLFragment appendValue(Float F) + { + if (null == F) + return appendNull(); + return appendValue(F.floatValue()); + } + + public SQLFragment appendValue(float f) + { + if (Float.isFinite(f)) + { + getStringBuilder().append(f); + } + else + { + getStringBuilder().append("?"); + add(f); + } + return this; + } + + public SQLFragment appendValue(Double D) + { + if (null == D) + return appendNull(); + else + return appendValue(D.doubleValue()); + } + + public SQLFragment appendValue(double d) + { + if (Double.isFinite(d)) + { + getStringBuilder().append(d); + } + else + { + getStringBuilder().append("?"); + add(d); + } + return this; + } + + public SQLFragment appendValue(Number N) + { + if (null == N) + return appendNull(); + + if (N instanceof Quantity q) + N = q.value(); + + if (N instanceof BigDecimal || N instanceof BigInteger || N instanceof Long) + { + getStringBuilder().append(N); + } + else if (Double.isFinite(N.doubleValue())) + { + getStringBuilder().append(N); + } + else + { + getStringBuilder().append(" ? "); + add(N); + } + return this; + } + + // Issue 27534: Stop using {fn now()} in function declarations + // Issue 48864: Query Table's use of web server time can cause discrepancies in created/modified timestamps + public final SQLFragment appendValue(NowTimestamp now) + { + if (null == now) + return appendNull(); + getStringBuilder().append("CURRENT_TIMESTAMP"); + return this; + } + + public final SQLFragment appendValue(java.util.Date d) + { + if (null == d) + return appendNull(); + if (d.getClass() == java.util.Date.class) + getStringBuilder().append("{ts '").append(new Timestamp(d.getTime())).append("'}"); + else if (d.getClass() == java.sql.Timestamp.class) + getStringBuilder().append("{ts '").append(d).append("'}"); + else if (d.getClass() == java.sql.Date.class) + getStringBuilder().append("{d '").append(d).append("'}"); + else + throw new IllegalStateException("Unexpected date type: " + d.getClass().getName()); + return this; + } + + public SQLFragment appendValue(GUID g) + { + return appendValue(g, null); + } + + public SQLFragment appendValue(GUID g, SqlDialect d) + { + if (null == g) + return appendNull(); + // doesn't need StringHandler, just hex and hyphen + String sqlGUID = "'" + g + "'"; + // I'm testing dialect type, because some dialects do not support getGuidType(), and postgers uses VARCHAR anyway + if (null != d && d.isSqlServer()) + getStringBuilder().append("CAST(").append(sqlGUID).append(" AS UNIQUEIDENTIFIER)"); + else + getStringBuilder().append(sqlGUID); + return this; + } + + public SQLFragment appendValue(Enum e) + { + if (null == e) + return appendNull(); + String name = e.name(); + // Enum.name() usually returns a simple string (a legal java identifier), this is a paranoia check. + if (name.contains("'")) + throw new IllegalStateException(); + getStringBuilder().append("'").append(name).append("'"); + return this; + } + + public SQLFragment append(FieldKey fk) + { + if (null == fk) + return appendNull(); + append(String.valueOf(fk)); + return this; + } + + + /** Adds the object as a JDBC parameter value */ + public SQLFragment add(Object p) + { + getMutableParams().add(p); + return this; + } + + + /** Adds the objects as JDBC parameter values */ + public SQLFragment addAll(Collection l) + { + getMutableParams().addAll(l); + return this; + } + + + /** Adds the objects as JDBC parameter values */ + public SQLFragment addAll(Object... values) + { + if (values == null) + return this; + addAll(Arrays.asList(values)); + return this; + } + + + /** Sets the parameter at the index to the object's value */ + public void set(int i, Object p) + { + getMutableParams().set(i,p); + } + + /** Append both the SQL and the parameters from the other SQLFragment to this SQLFragment */ + public SQLFragment append(SQLFragment f) + { + if (null != f.sb) + getStringBuilder().append(f.sb); + else + getStringBuilder().append(f.sql); + if (null != f.params) + addAll(f.params); + mergeCommonTableExpressions(f); + tempTokens.addAll(f.tempTokens); + return this; + } + + public SQLFragment append(@NotNull Iterable fragments, @NotNull String separator) + { + String s = ""; + for (SQLFragment fragment : fragments) + { + append(s); + s = separator; + append(fragment); + } + return this; + } + + // return boolean so this can be used in an assert. passing in a dialect is not ideal, but parsing comments out + // before submitting the fragment is not reliable and holding statements & comments separately (to eliminate the + // need to parse them) isn't particularly easy... so punt for now. + public boolean appendComment(String comment, SqlDialect dialect) + { + if (dialect.supportsComments()) + { + StringBuilder sb = getStringBuilder(); + int len = sb.length(); + if (len > 0 && sb.charAt(len-1) != '\n') + sb.append('\n'); + sb.append("\n-- "); + boolean truncated = comment.length() > 1000; + if (truncated) + comment = comment.substring(0,1000); + sb.append(comment); + if (StringUtils.countMatches(comment, "'")%2==1) + sb.append("'"); + if (truncated) + sb.append("..."); + sb.append('\n'); + } + return true; + } + + + /** see also append(TableInfo, String alias) */ + public SQLFragment append(TableInfo table) + { + SQLFragment s = table.getSQLName(); + if (s != null) + return append(s); + + String alias = table.getSqlDialect().makeLegalIdentifier(table.getName()); + return append(table.getFromSQL(alias)); + } + + /** Add a table/query to the SQL with an alias, as used in a FROM clause */ + public SQLFragment append(TableInfo table, String alias) + { + return append(table.getFromSQL(alias)); + } + + /** Add to the SQL */ + @Override + public SQLFragment append(char ch) + { + getStringBuilder().append(ch); + return this; + } + + /** This is like appendValue(CharSequence s), but force use of literal syntax + * CAUTIONARY NOTE: String literals in PostgresSQL are tricky because of overloaded functions + * array_agg('string') fails array_agg('string'::VARCHAR) works + * json_object('{}) works json_object('string'::VARCHAR) fails + * In the case of json_object() it expects TEXT. Postgres will promote 'json' to TEXT, but not 'json'::VARCHAR + */ + public SQLFragment appendStringLiteral(CharSequence s, @NotNull SqlDialect d) + { + if (null==s) + return appendNull(); + getStringBuilder().append(d.getStringHandler().quoteStringLiteral(s.toString())); + return this; + } + + /** Add to the SQL as either an in-line string literal or as a JDBC parameter depending on whether it would need escaping */ + public SQLFragment appendValue(CharSequence s) + { + return appendValue(s, null); + } + + public SQLFragment appendValue(CharSequence s, SqlDialect d) + { + if (null==s) + return appendNull(); + if (null==d || s.length() > 200) + return append("?").add(s.toString()); + appendStringLiteral(s, d); + return this; + } + + public SQLFragment appendInClause(@NotNull Collection params, SqlDialect dialect) + { + dialect.appendInClauseSql(this, params); + return this; + } + + public CharSequence getSqlCharSequence() + { + if (null != sb) + { + return sb; + } + return sql; + } + + public void insert(int index, SQLFragment sql) + { + if (!sql.getParams().isEmpty()) + { + throw new IllegalArgumentException("Not supported for SQLFragments with parameters - they must be inserted/merged separately"); + } + if (sql.commonTableExpressionsMap != null && !sql.commonTableExpressionsMap.isEmpty()) + { + throw new IllegalArgumentException("Not supported for SQLFragments with CTEs - they must be inserted/merged separately"); + } + if (!tempTokens.isEmpty()) + { + throw new IllegalArgumentException("Not supported for SQLFragments with temp tokens - they must be inserted/merged separately"); + } + getStringBuilder().insert(index, sql.getRawSQL()); + } + + /** Insert into the SQL */ + public void insert(int index, String str) + { + if ((StringUtils.countMatches(str, '\'') % 2) != 0 || + (StringUtils.countMatches(str, '\"') % 2) != 0 || + StringUtils.contains(str, ';')) + { + if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + throw new IllegalArgumentException("SQLFragment.insert(int,String) does not allow semicolons or unmatched quotes"); + } + + getStringBuilder().insert(index, str); + } + + /** Insert this SQLFragment's SQL and parameters at the start of the existing SQL and parameters */ + public void prepend(SQLFragment sql) + { + getStringBuilder().insert(0, sql.getSqlCharSequence().toString()); + if (null != sql.params) + getMutableParams().addAll(0, sql.params); + mergeCommonTableExpressions(sql); + } + + + public int indexOf(String str) + { + return getStringBuilder().indexOf(str); + } + + + // Display query in "English" (display SQL with params substituted) + // with a little more work could probably be made to be SQL legal + public String getFilterText() + { + String sql = getSQL().replaceFirst("WHERE ", ""); + List params = getParams(); + for (Object param1 : params) + { + String param = param1.toString(); + param = param.replaceAll("\\\\", "\\\\\\\\"); + param = param.replaceAll("\\$", "\\\\\\$"); + sql = sql.replaceFirst("\\?", param); + } + return sql.replaceAll("\"", ""); + } + + + @Override + public char charAt(int index) + { + return getSqlCharSequence().charAt(index); + } + + @Override + public int length() + { + return getSqlCharSequence().length(); + } + + @Override + public @NotNull CharSequence subSequence(int start, int end) + { + return getSqlCharSequence().subSequence(start, end); + } + + /** + * KEY is used as a faster way to look for equivalent CTE expressions. + * returning a name here allows us to potentially merge CTE at add time + * + * if you don't have a key you can just use sqlf.toString() + */ + public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf) + { + return addCommonTableExpression(dialect, key, proposedName, sqlf, false); + } + + public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf, boolean recursive) + { + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + CTE prev = commonTableExpressionsMap.get(key); + if (null != prev) + return prev.token(); + CTE cte = new CTE(dialect, proposedName, sqlf, recursive); + commonTableExpressionsMap.put(key, cte); + return cte.token(); + } + + public String createCommonTableExpressionToken(SqlDialect dialect, Object key, String proposedName) + { + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + CTE prev = commonTableExpressionsMap.get(key); + if (null != prev) + throw new IllegalStateException("Cannot create CTE token from already used key."); + CTE cte = new CTE(dialect ,proposedName); + commonTableExpressionsMap.put(key, cte); + return cte.token(); + } + + public void setCommonTableExpressionSql(Object key, SQLFragment sqlf, boolean recursive) + { + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + + if (null != sqlf.commonTableExpressionsMap && !sqlf.commonTableExpressionsMap.isEmpty()) + { + // Need to merge CTEs up; this.cte depends on newSql.ctes, so they need to come first + SQLFragment newSql = new SQLFragment(sqlf); + LinkedHashMap toMap = new LinkedHashMap<>(newSql.commonTableExpressionsMap); + for (Map.Entry e : commonTableExpressionsMap.entrySet()) + { + CTE from = e.getValue(); + CTE to = toMap.get(e.getKey()); + if (null != to) + to.tokens.addAll(from.tokens); + else + toMap.put(e.getKey(), from.copy(false)); + } + + commonTableExpressionsMap = toMap; + newSql.commonTableExpressionsMap = null; + sqlf = newSql; + } + + CTE cte = commonTableExpressionsMap.get(key); + if (null == cte) + throw new IllegalStateException("CTE not found."); + cte.sqlf = sqlf; + cte.recursive = recursive; + } + + + private void mergeCommonTableExpressions(SQLFragment sqlFrom) + { + if (null == sqlFrom.commonTableExpressionsMap || sqlFrom.commonTableExpressionsMap.isEmpty()) + return; + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + for (Map.Entry e : sqlFrom.commonTableExpressionsMap.entrySet()) + { + CTE from = e.getValue(); + CTE to = commonTableExpressionsMap.get(e.getKey()); + if (null != to) + to.tokens.addAll(from.tokens); + else + commonTableExpressionsMap.put(e.getKey(), from.copy(false)); + } + } + + + public void addTempToken(Object tempToken) + { + tempTokens.add(tempToken); + } + + public void addTempTokens(SQLFragment other) + { + tempTokens.add(other.tempTokens); + } + + public static SQLFragment prettyPrint(SQLFragment from) + { + SQLFragment sqlf = new SQLFragment(from); + + String s = from.getSqlCharSequence().toString(); + StringBuilder sb = new StringBuilder(s.length() + 200); + String[] lines = StringUtils.split(s, '\n'); + int indent = 0; + + for (String line : lines) + { + String t = line.trim(); + + if (t.isEmpty()) + continue; + + if (t.startsWith("-- params = b.getParams(); + assertEquals(2,params.size()); + assertEquals(5, params.get(0)); + assertEquals("xxyzzy", params.get(1)); + + + SQLFragment c = new SQLFragment(b); + assertEquals(""" + WITH + /*CTE*/ + \tCTE AS (SELECT a FROM b WHERE x=?) + SELECT * FROM CTE WHERE y=?""", + c.getSQL()); + assertEquals(""" + WITH + /*CTE*/ + \tCTE AS (SELECT a FROM b WHERE x=5) + SELECT * FROM CTE WHERE y='xxyzzy'""", + filterDebugString(c.toDebugString())); + params = c.getParams(); + assertEquals(2,params.size()); + assertEquals(5, params.get(0)); + assertEquals("xxyzzy", params.get(1)); + + + // combining + + SQLFragment sqlf = new SQLFragment(); + String token = sqlf.addCommonTableExpression(dialect, "KEY_A", "cte1", new SQLFragment("SELECT * FROM a")); + sqlf.append("SELECT * FROM ").append(token).append(" _1"); + + assertEquals(""" + WITH + /*CTE*/ + \tcte1 AS (SELECT * FROM a) + SELECT * FROM cte1 _1""", + sqlf.getSQL()); + + SQLFragment sqlf2 = new SQLFragment(); + String token2 = sqlf2.addCommonTableExpression(dialect, "KEY_A", "cte2", new SQLFragment("SELECT * FROM a")); + sqlf2.append("SELECT * FROM ").append(token2).append(" _2"); + assertEquals(""" + WITH + /*CTE*/ + \tcte2 AS (SELECT * FROM a) + SELECT * FROM cte2 _2""", + sqlf2.getSQL()); + + SQLFragment sqlf3 = new SQLFragment(); + String token3 = sqlf3.addCommonTableExpression(dialect, "KEY_B", "cte3", new SQLFragment("SELECT * FROM b")); + sqlf3.append("SELECT * FROM ").append(token3).append(" _3"); + assertEquals(""" + WITH + /*CTE*/ + \tcte3 AS (SELECT * FROM b) + SELECT * FROM cte3 _3""", + sqlf3.getSQL()); + + SQLFragment union = new SQLFragment(); + union.append(sqlf); + union.append("\nUNION\n"); + union.append(sqlf2); + union.append("\nUNION\n"); + union.append(sqlf3); + assertEquals(""" + WITH + /*CTE*/ + \tcte1 AS (SELECT * FROM a) + ,/*CTE*/ + \tcte3 AS (SELECT * FROM b) + SELECT * FROM cte1 _1 + UNION + SELECT * FROM cte1 _2 + UNION + SELECT * FROM cte3 _3""", + union.getSQL()); + } + + @Test + public void nested_cte() + { + // one-level cte using cteToken (CTE fragment 'a' does not contain a CTE) + { + SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); + assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); + SQLFragment b = new SQLFragment(); + String cteToken = b.addCommonTableExpression(dialect, new Object(), "CTE", a); + b.append("SELECT * FROM ").append(cteToken).append(" WHERE p=?").add("parameterTWO"); + assertEquals(""" + WITH + /*CTE*/ + \tCTE AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) + SELECT * FROM CTE WHERE p='parameterTWO'""", + filterDebugString(b.toDebugString())); + assertEquals("parameterONE", b.getParams().get(0)); + } + + // two-level cte using cteTokens (CTE fragment 'b' contains a CTE of fragment a) + { + SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); + assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); + SQLFragment b = new SQLFragment(); + String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); + b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterTWO"); + SQLFragment c = new SQLFragment(); + String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); + c.append("SELECT * FROM ").append(cteTokenB).append(" WHERE i=?").add(3); + assertEquals(""" + WITH + /*CTE*/ + \tA_ AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) + ,/*CTE*/ + \tB_ AS (SELECT * FROM A_ WHERE p='parameterTWO') + SELECT * FROM B_ WHERE i=3""", + filterDebugString(c.toDebugString())); + List params = c.getParams(); + assertEquals(3, params.size()); + assertEquals("parameterONE", params.get(0)); + assertEquals("parameterTWO", params.get(1)); + assertEquals(3, params.get(2)); + } + + // Same as previous but top-level query has both a nested and non-nested CTE + { + SQLFragment a = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); + SQLFragment a2 = new SQLFragment("SELECT 2 as i, 'Atwo' as s, CAST(? AS VARCHAR) as p", "parameterAtwo"); + SQLFragment b = new SQLFragment(); + String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); + b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); + SQLFragment c = new SQLFragment(); + String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); + String cteTokenA2 = c.addCommonTableExpression(dialect, new Object(), "A2_", a2); + c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); + assertEquals(""" + WITH + /*CTE*/ + \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) + ,/*CTE*/ + \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') + ,/*CTE*/ + \tA2_ AS (SELECT 2 as i, 'Atwo' as s, CAST('parameterAtwo' AS VARCHAR) as p) + SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", + filterDebugString(c.toDebugString())); + List params = c.getParams(); + assertEquals(4, params.size()); + assertEquals("parameterAone", params.get(0)); + assertEquals("parameterB", params.get(1)); + assertEquals("parameterAtwo", params.get(2)); + assertEquals(4, params.get(3)); + } + + // Same as previous but two of the CTEs are the same and should be collapsed (e.g. imagine a container filter implemented with a CTE) + // TODO, we only collapse CTEs that are siblings + { + SQLFragment cf = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); + SQLFragment b = new SQLFragment(); + String cteTokenA = b.addCommonTableExpression(dialect, "CTE_KEY_CF", "A_", cf); + b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); + SQLFragment c = new SQLFragment(); + String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); + String cteTokenA2 = c.addCommonTableExpression(dialect, "CTE_KEY_CF", "A2_", cf); + c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); + assertEquals(""" + WITH + /*CTE*/ + \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) + ,/*CTE*/ + \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') + ,/*CTE*/ + \tA2_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) + SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", + filterDebugString(c.toDebugString())); + List params = c.getParams(); + assertEquals(4, params.size()); + assertEquals("parameterAone", params.get(0)); + assertEquals("parameterB", params.get(1)); + assertEquals("parameterAone", params.get(2)); + assertEquals(4, params.get(3)); + } + } + + + private void shouldFail(Runnable r) + { + try + { + r.run(); + if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + fail("Expected IllegalArgumentException"); + } + catch (IllegalArgumentException e) + { + if (AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + fail("Did not expect IllegalArgumentException"); + } + } + + + @Test + public void testIllegalArgument() + { + shouldFail(() -> new SQLFragment(";")); + shouldFail(() -> new SQLFragment().append(";")); + shouldFail(() -> new SQLFragment("AND name='")); + shouldFail(() -> new SQLFragment().append("AND name = '")); + shouldFail(() -> new SQLFragment().append("AND name = 'Robert'); DROP TABLE Students; --")); + + shouldFail(() -> new SQLFragment().appendIdentifier("column name")); + shouldFail(() -> new SQLFragment().appendIdentifier("?")); + shouldFail(() -> new SQLFragment().appendIdentifier(";")); + shouldFail(() -> new SQLFragment().appendIdentifier("\"column\"name\"")); + } + + + String mysqlQuoteIdentifier(String id) + { + return "`" + id.replaceAll("`", "``") + "`"; + } + + @Test + public void testMysql() + { + // OK + new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("mysql")); + new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my`sql")); + new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my\"sql")); + + // not OK + shouldFail(() -> new SQLFragment().appendIdentifier("`")); + shouldFail(() -> new SQLFragment().appendIdentifier("`a`a`")); + } + } + + @Override + public boolean equals(Object obj) + { + if (!(obj instanceof SQLFragment other)) + { + return false; + } + return getSQL().equals(other.getSQL()) && getParams().equals(other.getParams()); + } + + /** + * Joins the SQLFragments in the provided {@code Iterable} into a single SQLFragment. The SQL is joined by string + * concatenation using the provided separator. The parameters are combined to form the new parameter list. + * + * @param fragments SQLFragments to join together + * @param separator Separator to use on the SQL portion + * @return A new SQLFragment that joins all the SQLFragments + */ + public static SQLFragment join(Iterable fragments, String separator) + { + if (separator.contains("?")) + throw new IllegalStateException("separator must not include a parameter marker"); + + // Join all the SQL statements + String sql = StreamSupport.stream(fragments.spliterator(), false) + .map(SQLFragment::getSQL) + .collect(Collectors.joining(separator)); + + // Collect all the parameters to a single list + List params = StreamSupport.stream(fragments.spliterator(), false) + .map(SQLFragment::getParams) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + + return new SQLFragment(sql, params); + } + + // Marker interface to hint that this value may be replaced by CURRENT_TIMESTAMP + public static class NowTimestamp extends java.sql.Timestamp + { + public NowTimestamp() + { + this(System.currentTimeMillis()); + } + + public NowTimestamp(long ms) + { + super(ms); + } + } +} diff --git a/api/src/org/labkey/api/data/StatementUtils.java b/api/src/org/labkey/api/data/StatementUtils.java index f6ae097188c..3fa83e62ede 100644 --- a/api/src/org/labkey/api/data/StatementUtils.java +++ b/api/src/org/labkey/api/data/StatementUtils.java @@ -1,1917 +1,1915 @@ -/* - * Copyright (c) 2012-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.data; - -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.CaseInsensitiveMapWrapper; -import org.labkey.api.collections.Sets; -import org.labkey.api.data.dialect.MockSqlDialect; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.dataiterator.SimpleTranslator; -import org.labkey.api.dataiterator.TableInsertUpdateDataIterator; -import org.labkey.api.exp.MvColumn; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.list.ListDefinition; -import org.labkey.api.exp.list.ListService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainKind; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.DomainUtil; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.gwt.client.model.GWTDomain; -import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; -import org.labkey.api.query.AliasManager; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryService; -import org.labkey.api.security.User; -import org.labkey.api.util.DateUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JunitUtil; -import org.labkey.api.util.TestContext; -import org.labkey.api.util.logging.LogHelper; - -import java.sql.Connection; -import java.sql.SQLException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Stream; - -import static java.util.Objects.requireNonNull; -import static org.labkey.api.util.JunitUtil.deleteTestContainer; - -public class StatementUtils -{ - private static final Logger _log = LogHelper.getLogger(StatementUtils.class, "SQL insert/update/delete generation"); - - public enum Operation {insert, update, merge} - - // configuration parameters - private Operation _operation; - private SqlDialect _dialect; - private final TableInfo _targetTable; - private Set _keyColumnNames = null; // override the primary key of _table - private Set _skipColumnNames = Set.of(); - private final Set _dontUpdateColumnNames = new CaseInsensitiveHashSet(); - private boolean _updateBuiltInColumns = false; // default to false, this should usually be handled by StandardDataIteratorBuilder - private boolean _selectIds = false; - private boolean _selectObjectUri = false; - private boolean _allowUpdateAutoIncrement = false; - private boolean _preferPKOverObjectUriAsKey = false; - - // variable/parameter tracking helpers - private boolean useVariables = false; - private final Map _constants = new CaseInsensitiveHashMap<>(); - final Map parameters = new CaseInsensitiveMapWrapper<>(new LinkedHashMap<>()); - - // ColumnTracker is used for test instrumentation and contains sets of column names that were included in - // given operations (insert, update, select) after running createStatement(). - // This is intended for test instrumentation use only. - private record ColumnTracker(Set insertColumns, Set updateColumns, Set selectColumns) - { - public ColumnTracker() - { - this(new CaseInsensitiveHashSet(), new CaseInsensitiveHashSet(), new CaseInsensitiveHashSet()); - } - } - - private ColumnTracker _columnTracker; - - // - // builder style methods - // - - //Vocabulary adhoc properties - private Set _vocabularyProperties = new HashSet<>(); - - public StatementUtils(@NotNull Operation op, @NotNull TableInfo table) - { - _operation = op; - _dialect = table.getSqlDialect(); - _targetTable = table; - } - - public StatementUtils dialect(SqlDialect dialect) - { - _dialect = dialect; - return this; - } - - public StatementUtils operation(@NotNull Operation op) - { - _operation = op; - return this; - } - - public StatementUtils constants(@NotNull Map constants) - { - _constants.putAll(constants); - return this; - } - - public StatementUtils keys(Set keyNames) - { - _keyColumnNames = keyNames; - return this; - } - - public StatementUtils skip(Set skip) - { - _skipColumnNames = null==skip ? Set.of() : skip; - return this; - } - - public StatementUtils noupdate(Set noupdate) - { - if (null != noupdate) - _dontUpdateColumnNames.addAll(noupdate); - return this; - } - - public StatementUtils updateBuiltinColumns(boolean b) - { - _updateBuiltInColumns = b; - return this; - } - - public StatementUtils selectIds(boolean b) - { - _selectIds = b; - return this; - } - - public StatementUtils selectObjectUri(boolean b) - { - _selectObjectUri = b; - return this; - } - - public StatementUtils allowSetAutoIncrement(boolean b) - { - _allowUpdateAutoIncrement = b; - return this; - } - - public StatementUtils setVocabularyProperties(Set vocabularyProperties) - { - _vocabularyProperties = vocabularyProperties; - return this; - } - - public StatementUtils setPreferPKOverObjectUriAsKey(boolean preferPKOverObjectUriAsKey) - { - _preferPKOverObjectUriAsKey = preferPKOverObjectUriAsKey; - return this; - } - - private static StatementUtils insertStatement(TableInfo table, boolean selectIds, boolean autoFillDefaultColumns) - { - return new StatementUtils(Operation.insert, table) - .updateBuiltinColumns(autoFillDefaultColumns) - .selectIds(selectIds); - } - - /** - * Create a reusable SQL Statement for inserting rows into a labkey relationship. The relationship - * persisted directly in the database (SchemaTableInfo), or via the OntologyManager tables. - *

- * QueryService shouldn't really know about the internals of exp.Object and exp.ObjectProperty etc. - * However, I can only keep so many levels of abstraction in my head at once. - *

- * NOTE: this is currently fairly expensive for updating one row into an Ontology stored relationship on Postgres. - * This shouldn't be a big problem since we don't usually need to optimize the one-row case, and we're moving - * to provisioned tables for major datatypes. - */ - public static ParameterMapStatement insertStatement(Connection conn, TableInfo table, @Nullable Container c, @Nullable User user, boolean selectIds, boolean autoFillDefaultColumns) throws SQLException - { - return insertStatement(table, selectIds, autoFillDefaultColumns) - .createStatement(conn, c, user); - } - - private static StatementUtils updateStatement(TableInfo table, boolean selectIds, boolean autoFillDefaultColumns) - { - return new StatementUtils(Operation.update, table) - .updateBuiltinColumns(autoFillDefaultColumns) - .selectIds(selectIds); - } - - /** - * Create a reusable SQL Statement for updating rows into a labkey relationship. The relationship - * persisted directly in the database (SchemaTableInfo), or via the OntologyManager tables. - *

- * QueryService shouldn't really know about the internals of exp.Object and exp.ObjectProperty etc. - * However, I can only keep so many levels of abstraction in my head at once. - *

- * NOTE: this is currently fairly expensive for updating one row into an Ontology stored relationship on Postgres. - * This shouldn't be a big problem since we don't usually need to optimize the one-row case, and we're moving - * to provisioned tables for major datatypes. - */ - public static ParameterMapStatement updateStatement(Connection conn, TableInfo table, @Nullable Container c, User user, boolean selectIds, boolean autoFillDefaultColumns) throws SQLException - { - return updateStatement(table, selectIds, autoFillDefaultColumns) - .createStatement(conn, c, user); - } - - private static StatementUtils mergeStatement(TableInfo table, @Nullable Set keyNames, @Nullable Set skipColumnNames, @Nullable Set dontUpdate, boolean selectIds, boolean autoFillDefaultColumns, boolean supportsAutoIncrementKey) - { - return new StatementUtils(Operation.merge, table) - .keys(keyNames) - .skip(skipColumnNames) - .allowSetAutoIncrement(supportsAutoIncrementKey) - .noupdate(dontUpdate) - .updateBuiltinColumns(autoFillDefaultColumns) - .selectIds(selectIds); - } - - public static ParameterMapStatement mergeStatement(Connection conn, TableInfo table, @Nullable Set keyNames, @Nullable Set skipColumnNames, @Nullable Set dontUpdate, @Nullable Container c, @Nullable User user, boolean selectIds, boolean autoFillDefaultColumns, boolean supportsAutoIncrementKey) throws SQLException - { - return mergeStatement(table, keyNames, skipColumnNames, dontUpdate, selectIds, autoFillDefaultColumns, supportsAutoIncrementKey) - .createStatement(conn, c, user); - } - - /* - * Parameter and Variable helpers - */ - - private static class ParameterHolder - { - ParameterHolder(Parameter p) - { - this.p = p; - _columnInfo = null; - } - - ParameterHolder(Parameter p, ColumnInfo c) - { - this.p = p; - _columnInfo = c; - } - - int getScale() - { - var type = requireNonNull(p.getType()); - if (null == _columnInfo || _columnInfo.getScale() <= 0) - return -1; - // GUID.isText()==true - if (JdbcType.GUID != type && (type.isText() || type.isDecimal())) - return _columnInfo.getScale(); - return -1; - } - - int getPrecision() - { - return null==_columnInfo ? -1 : _columnInfo.getPrecision(); - } - - final Parameter p; - final ColumnInfo _columnInfo; - String variableName = null; - Object constantValue = null; - boolean isConstant = false; - } - - private final static String pgRowVarPrefix = "$1."; - private String makeVariableName(String name) - { - String shortName = StringUtils.substring(name,0,32); // name is just for readability, make it short - String uniquePrefix = (_dialect.isSqlServer() ? "@" : pgRowVarPrefix) + ("p" + (parameters.size()+1) + "_"); - return uniquePrefix + AliasManager.makeLegalName(shortName, _dialect, true, uniquePrefix.length()); - } - - private String makePgRowTypeName(String variableName) - { - return StringUtils.substringAfter(variableName, pgRowVarPrefix); - } - - private ParameterHolder createParameter(ColumnInfo c) - { - ParameterHolder ph = parameters.get(c.getName()); - if (null == ph) - { - ph = new ParameterHolder(new Parameter(c.getName(), c.getPropertyURI(), null, c.getJdbcType()), c); - // NOTE: earlier DataIterator should probably split file into two columns: attachment_name, attachment_body - if (c.getInputType().equalsIgnoreCase("file") && c.getJdbcType() == JdbcType.VARCHAR) - ph.p.setFileAsName(true); - initParameterHolder(ph); - parameters.put(c.getName(), ph); - } - return ph; - } - - private void initParameterHolder(ParameterHolder ph) - { - String name = ph.p.getName(); - JdbcType type = ph.p.getType(); - assert null != type; - if (_constants.containsKey(name)) - { - Object value = Parameter.getValueToBind(_constants.get(name), type); - if (null == value || value instanceof Number || value instanceof String || value instanceof java.util.Date) - { - ph.isConstant = true; - ph.constantValue = value; - } - } - ph.variableName = makeVariableName(name); - } - - - private ParameterHolder createParameter(String name, JdbcType type) - { - ParameterHolder ph = parameters.get(name); - if (null == ph) - { - ph = new ParameterHolder(new Parameter(name, type)); - initParameterHolder(ph); - parameters.put(name, ph); - } - return ph; - } - - - private ParameterHolder createParameter(String name, String uri, JdbcType type) - { - ParameterHolder ph = parameters.get(name); - if (null == ph) - { - ph = new ParameterHolder(new Parameter(name, uri, null, type)); - initParameterHolder(ph); - parameters.put(name, ph); - } - return ph; - } - - private SQLFragment appendParameterOrVariable(SQLFragment f, ParameterHolder ph) - { - if (ph.isConstant) - { - toLiteral(f, ph.constantValue); - } - else if (useVariables) - { - f.append(ph.variableName); - } - else - { - f.append("?"); - f.add(ph.p); - } - return f; - } - - private SQLFragment appendPropertyValue(SQLFragment f, DomainProperty dp, ParameterHolder p) - { - if (dp.getJdbcType() == JdbcType.BOOLEAN) - { - f.append("CASE CAST("); - appendParameterOrVariable(f, p); - f.append(" AS ").append(_dialect.getBooleanDataType()).append(")") - .append(" WHEN ").append(_dialect.getBooleanTRUE()).append(" THEN 1.0 ") - .append(" WHEN ").append(_dialect.getBooleanFALSE()).append(" THEN 0.0 ") - .append(" ELSE NULL END"); - return f; - } - else - { - return appendParameterOrVariable(f, p); - } - } - - private void appendSQLFObjectProperty(SQLFragment sqlfObjectProperty, DomainProperty dp, String objectIdVar, String ifTHEN, String ifEND) - { - PropertyType propertyType = dp.getPropertyDescriptor().getPropertyType(); - ParameterHolder v = createParameter(dp.getName(), dp.getPropertyURI(), propertyType.getJdbcType()); - ParameterHolder mv = createParameter(dp.getName()+ MvColumn.MV_INDICATOR_SUFFIX, dp.getPropertyURI() + MvColumn.MV_INDICATOR_SUFFIX, JdbcType.VARCHAR); - sqlfObjectProperty.append("IF ("); - appendPropertyValue(sqlfObjectProperty, dp, v); - sqlfObjectProperty.append(" IS NOT NULL"); - if (dp.isMvEnabled()) - { - sqlfObjectProperty.append(" OR "); - appendParameterOrVariable(sqlfObjectProperty, mv); - sqlfObjectProperty.append(" IS NOT NULL"); - } - sqlfObjectProperty.append(")"); - sqlfObjectProperty.append(ifTHEN); - sqlfObjectProperty.append("INSERT INTO exp.ObjectProperty (objectid, propertyid, typetag, mvindicator, "); - sqlfObjectProperty.append(propertyType.getValueTypeColumn()); - sqlfObjectProperty.append(") VALUES ("); - sqlfObjectProperty.append(objectIdVar); - sqlfObjectProperty.append(",").appendValue(dp.getPropertyId()); - sqlfObjectProperty.append(",").appendStringLiteral(String.valueOf(propertyType.getStorageType()), _dialect); - sqlfObjectProperty.append(","); - if (dp.isMvEnabled()) - appendParameterOrVariable(sqlfObjectProperty, mv); - else - sqlfObjectProperty.append("NULL"); - sqlfObjectProperty.append(","); - appendPropertyValue(sqlfObjectProperty, dp, v); - sqlfObjectProperty.append(")").appendEOS(); - sqlfObjectProperty.append(ifEND); - sqlfObjectProperty.appendEOS(); - } - - private void appendSQLFDeleteObjectProperty(SQLFragment sqlfDelete, String objectIdVar, List domainProperties, Set vocabularyProperties) - { - var properties = null == domainProperties ? vocabularyProperties : domainProperties; - sqlfDelete.append("DELETE FROM exp.ObjectProperty WHERE ObjectId = "); - sqlfDelete.append(objectIdVar); - sqlfDelete.append(" AND PropertyId IN ("); - String separator = ""; - for (DomainProperty property : properties) - { - sqlfDelete.append(separator); - separator = ", "; - sqlfDelete.appendValue(property.getPropertyId()); - } - sqlfDelete.append(")").appendEOS(); - } - - private void setObjectUriPreselect(SQLFragment sqlfPreselectObject, TableInfo table, LinkedHashMap keys, String objectURIVar, String objectURIColumnName, ParameterHolder objecturiParameter) - { - String setKeyword = _dialect.isPostgreSQL() ? "" : "SET "; - if (Operation.merge == _operation || Operation.update == _operation) - { - // this seems overkill actually, but I'm focused on optimizing insert right now (MAB) - sqlfPreselectObject.append(setKeyword).append(objectURIVar).append(" = COALESCE(("); - sqlfPreselectObject.append("SELECT ").appendIdentifier(table.getColumn(objectURIColumnName).getSelectIdentifier()); - sqlfPreselectObject.append(" FROM ").append(table.getSQLName()); - sqlfPreselectObject.append(getPkWhereClause(keys)); - sqlfPreselectObject.append("),"); - appendParameterOrVariable(sqlfPreselectObject, objecturiParameter); - sqlfPreselectObject.append(")").appendEOS(); - - } - else - { - sqlfPreselectObject.append(setKeyword).append(objectURIVar).append(" = "); - appendParameterOrVariable(sqlfPreselectObject, objecturiParameter); - sqlfPreselectObject.appendEOS(); - } - } - - public ParameterMapStatement createStatement(Connection conn, @Nullable Container c, User user) throws SQLException - { - ParameterMapStatement statement = null; - try - { - statement = createStatement(conn, c, user, false); - } - catch (TableInsertUpdateDataIterator.NoUpdatableColumnInDataException e) - { - // ignore error - } - return statement; - } - - public ParameterMapStatement createStatement(Connection conn, @Nullable Container c, User user, boolean checkUpdatableColumns) throws SQLException, TableInsertUpdateDataIterator.NoUpdatableColumnInDataException - { - if (!(_targetTable instanceof UpdateableTableInfo updatable)) - throw new IllegalArgumentException("Table must be an UpdateableTableInfo"); - - TableInfo table = updatable.getSchemaTableInfo(); - - if (table.getTableType() != DatabaseTableType.TABLE) - throw new IllegalArgumentException("Table must be a database table"); - if (null == table.getMetaDataIdentifier()) - throw new IllegalArgumentException("Table must have a metadata identifier"); - - if (Operation.merge == _operation) - { - if (!_dialect.isPostgreSQL() && !_dialect.isSqlServer()) - throw new IllegalArgumentException("Merge is only supported/tested on postgres and sql server"); - } - - useVariables = Operation.merge == _operation; - String ifTHEN = _dialect.isSqlServer() ? " BEGIN " : " THEN "; - String ifEND = _dialect.isSqlServer() ? " END " : " END IF "; - - if (null != c) - { - assert null == _constants.get("container") || c.getId().equals(_constants.get("container")); - if (null == _constants.get("container")) - _constants.put("container", c.getId()); - } - - String objectURIColumnName = updatable.getObjectUriType() == UpdateableTableInfo.ObjectUriType.schemaColumn - ? updatable.getObjectURIColumnName() - : "objecturi"; - ParameterHolder objecturiParameter = null; - if (null != objectURIColumnName) - objecturiParameter = createParameter(objectURIColumnName, JdbcType.VARCHAR); - - // - // Keys for UPDATE or MERGE - // - LinkedHashMap keys = getKeys(updatable, table, objectURIColumnName, _keyColumnNames, _preferPKOverObjectUriAsKey); - - // - // exp.Objects INSERT - // - - SQLFragment sqlfDeclare = new SQLFragment(); - SQLFragment sqlfPreselectObject = new SQLFragment(); - SQLFragment sqlfInsertObject = new SQLFragment(); - SQLFragment sqlfSelectObject = new SQLFragment(); - SQLFragment sqlfObjectProperty = new SQLFragment(); - SQLFragment sqlfDelete = new SQLFragment(); - - Domain domain = updatable.getDomain(); - DomainKind domainKind = updatable.getDomainKind(); - List properties = Collections.emptyList(); - - boolean hasObjectURIColumn = objectURIColumnName != null && table.getColumn(objectURIColumnName) != null; - boolean alwaysInsertExpObject = (hasObjectURIColumn && updatable.isAlwaysInsertExpObject()) && Operation.update != _operation; - if (hasObjectURIColumn) - _dontUpdateColumnNames.add(objectURIColumnName); -// TODO Should we add created and createdby? Or make the caller decide? - if (Operation.update == _operation) - { - _dontUpdateColumnNames.add("Created"); - _dontUpdateColumnNames.add("CreatedBy"); - } - - String objectIdVar = null; - String objectURIVar = null; - boolean objectUriPreselectSet = false; - boolean isMaterializedDomain = null != domain && null != domainKind && StringUtils.isNotEmpty(domainKind.getStorageSchemaName()); - if (alwaysInsertExpObject || (null != domain && !isMaterializedDomain) || !_vocabularyProperties.isEmpty()) - { - properties = (null==domain||isMaterializedDomain) ? Collections.emptyList() : domain.getProperties(); - - if (alwaysInsertExpObject || !properties.isEmpty() || !_vocabularyProperties.isEmpty()) - { - if (!_dialect.isPostgreSQL() && !_dialect.isSqlServer()) - throw new IllegalStateException("Domains are only supported for sql server and postgres"); - - objectIdVar = _dialect.isPostgreSQL() ? "_$objectid$_" : "@_objectid_"; - sqlfDeclare.append("DECLARE ").append(objectIdVar).append(" BIGINT").appendEOS(); - objectURIVar = _dialect.isPostgreSQL() ? "_$objecturi$_" : "@_objecturi_"; - sqlfDeclare.append("DECLARE ").append(objectURIVar).append(" ").append(_dialect.getSqlTypeName(JdbcType.VARCHAR)).append("(300)").appendEOS(); - useVariables |= _dialect.isPostgreSQL(); - - ParameterHolder containerParameter = createParameter("container", JdbcType.GUID); - - // Insert a new row in exp.Object if there isn't already a row for this object - - // Grab the object's ObjectId based on the pk of the base table - if (hasObjectURIColumn || !_vocabularyProperties.isEmpty()) - { - setObjectUriPreselect(sqlfPreselectObject, table, keys, objectURIVar, objectURIColumnName, objecturiParameter); - objectUriPreselectSet = true; - } - - SQLFragment sqlfWhereObjectURI = new SQLFragment(); - sqlfWhereObjectURI.append("(ObjectURI = ").append(objectURIVar).append(")"); - - // In the update case, it's still possible that there isn't a row in exp.Object - there might have been - // no properties in the domain when the row was originally inserted - sqlfInsertObject.append("INSERT INTO exp.Object (container, objecturi, ownerobjectid) "); - sqlfInsertObject.append("SELECT "); - appendParameterOrVariable(sqlfInsertObject, containerParameter); - sqlfInsertObject.append(" AS Container,"); - appendParameterOrVariable(sqlfInsertObject, objecturiParameter); - sqlfInsertObject.append(" AS ObjectURI, "); - Long ownerObjectId = updatable.getOwnerObjectId(); - sqlfInsertObject.append( null == ownerObjectId ? "NULL" : String.valueOf(ownerObjectId) ).append(" AS OwnerObjectId"); - sqlfInsertObject.append(" WHERE NOT EXISTS (SELECT ObjectURI FROM exp.Object WHERE Container = "); - appendParameterOrVariable(sqlfInsertObject, containerParameter); - sqlfInsertObject.append(" AND ").append(sqlfWhereObjectURI).append(")").appendEOS(); - - // re-grab the object's ObjectId, in case it was just inserted - sqlfSelectObject.append(_dialect.isPostgreSQL() ? "" : "SET ").append(objectIdVar).append(" = ("); - sqlfSelectObject.append("SELECT ObjectId FROM exp.Object WHERE Container = "); - appendParameterOrVariable(sqlfSelectObject, containerParameter); - sqlfSelectObject.append(" AND ").append(sqlfWhereObjectURI).append(")").appendEOS(); - - if (Operation.insert != _operation && (!properties.isEmpty() || !_vocabularyProperties.isEmpty())) - { - // Clear out any existing property values for this domain - if (!properties.isEmpty()) - { - appendSQLFDeleteObjectProperty(sqlfDelete, objectIdVar, properties, null); - } - - // Clear out any existing ad hoc property - if (!_vocabularyProperties.isEmpty()) - { - appendSQLFDeleteObjectProperty(sqlfDelete, objectIdVar, null, _vocabularyProperties); - } - } - } - } - - if (_selectObjectUri) - { - if (objectURIVar == null) - { - objectURIVar = _dialect.isPostgreSQL() ? "_$objecturi$_" : "@_objecturi_"; - sqlfDeclare.append("DECLARE ").append(objectURIVar).append(" ").append(_dialect.getSqlTypeName(JdbcType.VARCHAR)).append("(300)").appendEOS(); - } - - if (!objectUriPreselectSet && (hasObjectURIColumn || !_vocabularyProperties.isEmpty())) - { - setObjectUriPreselect(sqlfPreselectObject, table, keys, objectURIVar, objectURIColumnName, objecturiParameter); - } - } - - - // - // BASE TABLE INSERT() - // - - ColumnInfo col; - List cols = new ArrayList<>(); - List values = new ArrayList<>(); - Set done = Sets.newCaseInsensitiveHashSet(); - - if (_updateBuiltInColumns && Operation.update != _operation) - { - col = table.getColumn("Owner"); - if (null != col && null != user) - { - cols.add(col); - values.add(new SQLFragment().appendValue(user.getUserId())); - done.add("Owner"); - } - col = table.getColumn("CreatedBy"); - if (null != col && null != user) - { - cols.add(col); - values.add(new SQLFragment().appendValue(user.getUserId())); - done.add("CreatedBy"); - } - col = table.getColumn("Created"); - if (null != col) - { - cols.add(col); - values.add(new SQLFragment().appendValue(new SQLFragment.NowTimestamp())); - done.add("Created"); - } - } - - ColumnInfo colModifiedBy = table.getColumn("ModifiedBy"); - if (_updateBuiltInColumns && null != colModifiedBy && null != user) - { - cols.add(colModifiedBy); - values.add(new SQLFragment().appendValue(user.getUserId())); - done.add("ModifiedBy"); - } - - ColumnInfo colModified = table.getColumn("Modified"); - if (_updateBuiltInColumns && null != colModified) - { - cols.add(colModified); - values.add(new SQLFragment().appendValue(new SQLFragment.NowTimestamp())); - done.add("Modified"); - } - ColumnInfo colVersion = table.getVersionColumn(); - if (_updateBuiltInColumns && null != colVersion && !done.contains(colVersion.getName())) - { - SQLFragment expr = colVersion.getVersionUpdateExpression(); - if (null != expr) - { - cols.add(colVersion); - values.add(expr); - done.add(colVersion.getName()); - } - } - - String objectIdColumnName = StringUtils.trimToNull(updatable.getObjectIdColumnName()); - ColumnInfo autoIncrementColumn = null; - CaseInsensitiveHashMap remap = updatable.remapSchemaColumns(); - if (null == remap) - remap = CaseInsensitiveHashMap.of(); - - for (ColumnInfo column : table.getColumns()) - { - if (column instanceof WrappedColumn || column.isCalculated()) - continue; - // if we're allowing the caller to set the auto-increment column, then treat like a regular column - if (column.isAutoIncrement() && !_allowUpdateAutoIncrement) - { - autoIncrementColumn = column; - continue; - } - if (column.isVersionColumn() && column != colModified) - continue; - String name = column.getName(); - if (done.contains(name)) - continue; - done.add(name); - ColumnInfo updatableColumn = updatable.getColumn(column.getName()); - if (updatableColumn != null && updatableColumn.hasDbSequence()) - _dontUpdateColumnNames.add(column.getName()); - - SQLFragment valueSQL = new SQLFragment(); - if (column.getName().equalsIgnoreCase(objectIdColumnName)) - { - valueSQL.append(objectIdVar); - } - else if (column.getName().equalsIgnoreCase(updatable.getObjectURIColumnName()) && null != objecturiParameter) - { - appendParameterOrVariable(valueSQL, objecturiParameter); - } - else - { - if (null != _skipColumnNames && _skipColumnNames.contains(Objects.toString(remap.get(name),name))) - continue; - ParameterHolder ph = createParameter(column); - appendParameterOrVariable(valueSQL, ph); - } - cols.add(column); - values.add(valueSQL); - } - - boolean selectAutoIncrement = false; - - assert cols.size() == values.size() : cols.size() + " columns and " + values.size() + " values - should match"; - - // - // INSERT - // - - String comma; - String rowIdVar = null; - SQLFragment sqlfInsertInto = new SQLFragment(); - - // Construct a new column tracker for test instrumentation - _columnTracker = new ColumnTracker(); - - if (Operation.insert == _operation || Operation.merge == _operation) - { - // Create a standard INSERT INTO table (col1, col2) VALUES (val1, val2) statement - // or (for degenerate, empty values case) INSERT INTO table VALUES (DEFAULT) - sqlfInsertInto.append("INSERT INTO ").append(table.getSQLName()); - - if (values.isEmpty()) - { - sqlfInsertInto.append("\nVALUES (DEFAULT)"); - } - else - { - sqlfInsertInto.append(" ("); - comma = ""; - for (ColumnInfo colInfo : cols) - { - sqlfInsertInto.append(comma); - comma = ", "; - sqlfInsertInto.appendIdentifier(colInfo.getSelectIdentifier()); - _columnTracker.insertColumns.add(colInfo.getName()); - } - sqlfInsertInto.append(")"); - - sqlfInsertInto.append("\nSELECT "); - comma = ""; - for (SQLFragment valueSQL : values) - { - sqlfInsertInto.append(comma); - comma = ", "; - sqlfInsertInto.append(valueSQL); - } - } - - if (_selectIds && null != autoIncrementColumn) - { - selectAutoIncrement = true; - if (useVariables) - rowIdVar = "_rowid_"; - rowIdVar = _dialect.addReselect(sqlfInsertInto, autoIncrementColumn, rowIdVar); - if (useVariables) - sqlfDeclare.append("DECLARE ").append(rowIdVar).append(" BIGINT").appendEOS(); // CONSIDER: Move this into addReselect()? - } - - if (_selectObjectUri && hasObjectURIColumn) - { - _dialect.addReselect(sqlfInsertInto, table.getColumn(objectURIColumnName), objectURIVar); - } - } - - // - // UPDATE - // - - SQLFragment sqlfUpdate = new SQLFragment(); - if (Operation.update == _operation || Operation.merge == _operation) - { - // Create a standard UPDATE table SET col1 = val1, col2 = val2 statement - sqlfUpdate.append("UPDATE ").append(table.getSQLName()).append("\nSET "); - comma = ""; - int updateCount = 0; - for (int i = 0; i < cols.size(); i++) - { - col = cols.get(i); - FieldKey fk = col.getFieldKey(); - if (keys.containsKey(fk)) - continue; - - // Issue 52666: Check column remapping when looking for columns to not update - String colName = col.getName(); - if (_dontUpdateColumnNames.contains(colName) || (remap.containsKey(colName) && _dontUpdateColumnNames.contains(remap.get(colName)))) - continue; - - sqlfUpdate.append(comma); - comma = ", "; - sqlfUpdate.appendIdentifier(col.getSelectIdentifier()); - sqlfUpdate.append(" = "); - sqlfUpdate.append(values.get(i)); - _columnTracker.updateColumns.add(col.getName()); - updateCount++; - } - - if (Operation.update == _operation && updateCount == 0) - { - if (checkUpdatableColumns) - throw new TableInsertUpdateDataIterator.NoUpdatableColumnInDataException(table.getName()); - - sqlfUpdate.appendIdentifier(keys.values().iterator().next().getSelectIdentifier()); - sqlfUpdate.append(" = 'noop' WHERE 1 <> 1").appendEOS(); - } - else - { - sqlfUpdate.append(getPkWhereClause(keys)); - sqlfUpdate.appendEOS(); - } - - if (Operation.merge == _operation) - { - // updateCount can equal 0. This happens particularly when inserting into junction tables where - // there are two columns and both are in the primary key - if (0 == updateCount) - { - sqlfUpdate = new SQLFragment(); - sqlfInsertInto.append("\nWHERE NOT EXISTS (SELECT * FROM ").append(table.getSQLName()); - sqlfInsertInto.append(getPkWhereClause(keys)); - sqlfInsertInto.append(")"); - } - else - { - sqlfUpdate.append("IF "); - sqlfUpdate.append(_dialect.isSqlServer() ? "@@ROWCOUNT=0" : "NOT FOUND"); - sqlfUpdate.append(ifTHEN).append("\n\t"); - - sqlfInsertInto.appendEOS(); - sqlfInsertInto.append(ifEND); - } - } - } - - if (Operation.insert == _operation || Operation.merge == _operation) - sqlfInsertInto.appendEOS(); - - SQLFragment sqlfSelectIds = null; - - if ((_selectIds && (null != objectIdVar || null != rowIdVar)) || (_selectObjectUri && null != objectURIVar)) - { - sqlfSelectIds = new SQLFragment("SELECT "); - comma = ""; - if (_selectIds) - { - if (null != rowIdVar) - { - sqlfSelectIds.append(rowIdVar); - _columnTracker.selectColumns.add(rowIdVar); - comma = ","; - } - if (null != objectIdVar) - { - sqlfSelectIds.append(comma).append(objectIdVar); - _columnTracker.selectColumns.add(objectIdVar); - comma = ","; - } - } - - if (_selectObjectUri && null != objectURIVar) - { - sqlfSelectIds.append(comma).append(objectURIVar); - _columnTracker.selectColumns.add(objectIdVar); - } - } - - // - // ObjectProperty - // - - if (!properties.isEmpty()) - { - Set skip = updatable.skipProperties(); - if (null != skip) - done.addAll(skip); - - for (DomainProperty dp : properties) - { - // ignore property that 'wraps' a hard column - if (done.contains(dp.getName())) - continue; - appendSQLFObjectProperty(sqlfObjectProperty, dp, objectIdVar, ifTHEN, ifEND); - } - } - - if (!_vocabularyProperties.isEmpty()) - { - for (DomainProperty vocProp: _vocabularyProperties) - { - appendSQLFObjectProperty(sqlfObjectProperty, vocProp, objectIdVar, ifTHEN, ifEND); - } - } - - // - // PREPARE - // - - ParameterMapStatement ret; - - if (!useVariables) - { - SQLFragment script = new SQLFragment(); - Stream.of(sqlfDeclare, sqlfPreselectObject, sqlfInsertObject, sqlfSelectObject, sqlfDelete, sqlfUpdate, sqlfInsertInto, sqlfObjectProperty, sqlfSelectIds) - .filter(f -> null != f && !f.isEmpty()) - .forEach(script::append); - ret = new ParameterMapStatement(table.getSchema().getScope(), conn, script, remap); - } - else if (_dialect.isSqlServer()) - { - if (!parameters.isEmpty()) - { - SQLFragment select = new SQLFragment(); - sqlfDeclare.append("DECLARE "); - select.append("SELECT "); - comma = ""; - for (Map.Entry e : parameters.entrySet()) - { - ParameterHolder ph = e.getValue(); - sqlfDeclare.append(comma); - String variable = sqlServerVariableDeclaration(sqlfDeclare, ph); - select.append(comma).append(variable).append("=?"); - select.add(ph.p); - comma = ", "; - } - sqlfDeclare.appendEOS(); - sqlfDeclare.append(select); - sqlfDeclare.appendEOS(); - } - SQLFragment script = new SQLFragment(); - Stream.of(sqlfDeclare, sqlfPreselectObject, sqlfInsertObject, sqlfSelectObject, sqlfDelete, sqlfUpdate, sqlfInsertInto, sqlfObjectProperty, sqlfSelectIds) - .filter(f -> null != f && !f.isEmpty()) - .forEach(script::append); - _log.debug(script.toDebugString()); - ret = new ParameterMapStatement(table.getSchema().getScope(), conn, script, remap); - } - else - { - // wrap in a function with a single ROW() constructor argument - SQLFragment fn = new SQLFragment(); - String fnName = _dialect.getGlobalTempTablePrefix() + "fn_" + GUID.makeHash(); - TempTableTracker.track(fnName, fn); - - String typeName = fnName + "type"; - fn.append("CREATE TYPE ").append(typeName).append(" AS ("); - // TODO d.execute() doesn't handle temp schema - SQLFragment call = new SQLFragment(); - call.append(fnName).append("(ROW("); - comma = ""; - for (Map.Entry e : parameters.entrySet()) - { - ParameterHolder ph = e.getValue(); - String type = _dialect.getSqlTypeName(ph.p.getType()); - fn.append("\n").append(comma); - fn.append(makePgRowTypeName(ph.variableName)); - fn.append(" "); - fn.append(type); - // For PG (29687) we need the length for CHAR type - if (_dialect.isPostgreSQL() && JdbcType.CHAR.equals(ph.p.getType())) - fn.append("(").appendValue(ph.getScale()).append(")"); - call.append(comma).append("?"); - call.add(ph.p); - comma = ","; - } - fn.append("\n)").appendEOS(); - fn.append("CREATE FUNCTION ").append(fnName).append("(").append(typeName).append(") "); - fn.append("RETURNS "); - if (null != sqlfSelectIds) - fn.append("SETOF RECORD"); - else - fn.append("void"); - String quoteToken = "$x" + GUID.makeHash() + "$"; - fn.append(" AS ").append(quoteToken).append("\n"); - call.append("))"); - - if (null != sqlfSelectIds) - { - call.insert(0, "SELECT * FROM "); - call.append(" AS x("); - String sep = ""; - - if (_selectIds) - { - if (null != rowIdVar) - { - call.append("A BIGINT"); - sep = ", "; - } - if (null != objectIdVar) - { - call.append(sep); - call.append("B BIGINT"); - sep = ", "; - } - } - - if (_selectObjectUri && null != objectURIVar) - { - call.append(sep); - call.append("C VARCHAR"); - } - - call.append(")").appendEOS(); - } - else - { - call.insert(0, "{call "); - call.append("}"); - } - - fn.append(sqlfDeclare); - - fn.append("BEGIN\n"); - fn.append("-- ").append(_operation.name()).append("\n"); - Stream.of(sqlfPreselectObject, sqlfInsertObject, sqlfSelectObject, sqlfDelete, sqlfUpdate, sqlfInsertInto, sqlfObjectProperty) - .filter(f -> null != f && !f.isEmpty()) - .forEach(fn::append); - if (null == sqlfSelectIds) - { - fn.append("RETURN").appendEOS(); - } - else - { - sqlfSelectIds.insert(0, "RETURN QUERY\n"); - fn.append(sqlfSelectIds); - fn.appendEOS(); - } - fn.append("END").appendEOS().append(" ").append(quoteToken).append(" LANGUAGE plpgsql").appendEOS(); - _log.debug(fn.toDebugString()); - _log.debug(call.toDebugString()); - final SQLFragment drop = new SQLFragment("DROP TYPE IF EXISTS ").append(typeName).append(" CASCADE").appendEOS(); - _log.debug(drop.toDebugString()); - new SqlExecutor(table.getSchema()).execute(fn); - ret = new ParameterMapStatement(table.getSchema().getScope(), conn, call, updatable.remapSchemaColumns()); - ret.setDebugSql(fn.getSQL() + "--\n" + call.toDebugString()); - ret.onClose(() -> { - try - { - new SqlExecutor(ExperimentService.get().getSchema()).execute(drop); - } - catch (Exception x) - { - _log.error("Error dropping custom rowtype for temp function.", x); - } - }); - } - - int selectIndex = 1; - - if (_selectIds) - { - // Why is one of these boolean and the other an index?? I don't know - ret.setSelectRowId(selectAutoIncrement); - - if (selectAutoIncrement) - selectIndex++; - - if (null != objectIdVar) - ret.setObjectIdIndex(selectIndex++); - } - - if (_selectObjectUri && null != objectURIVar) - ret.setObjectUriIndex(selectIndex); - - return ret; - } - - private static LinkedHashMap getKeys( - UpdateableTableInfo updatable, - TableInfo table, - String objectURIColumnName, - Set keyColumnNames, - boolean preferPKOverObjectUriAsKey - ) - { - LinkedHashMap keys = new LinkedHashMap<>(); - ColumnInfo col = table.getColumn("Container"); - - if (null != col) - keys.put(col.getFieldKey(), col); - - if (null != keyColumnNames && !keyColumnNames.isEmpty()) - { - for (String name : keyColumnNames) - { - col = table.getColumn(name); - if (null == col) - throw new IllegalArgumentException("Column not found: " + name); - keys.put(col.getFieldKey(), col); - } - } - else - { - // using objectURIColumnName preferentially to be backward compatible with OntologyManager.saveTabDelimited - // which in turn is only called by LuminexDataHandler.saveDataRows() - col = objectURIColumnName == null ? null : table.getColumn(objectURIColumnName); - if (null != col && !preferPKOverObjectUriAsKey) - keys.put(col.getFieldKey(), col); - else - { - // See Issue 26661 and Issue 41053 - // NOTE: IMO we should not be using updatable.getPkColumnNames() here! If the caller doesn't want to use the - // 'real' PK from the SchemaTableInfo for update/merge, then the alternate keys should be explicitly specified - // using StatementUtils.keys() - for (String pkName : updatable.getPkColumnNames()) - { - col = table.getColumn(pkName); - if (null == col) - throw new IllegalStateException("pk column not found: " + pkName); - keys.put(col.getFieldKey(), col); - } - } - } - - return keys; - } - - private SQLFragment getPkWhereClause(LinkedHashMap keys) - { - SQLFragment sqlfWherePK = new SQLFragment(); - sqlfWherePK.append("\nWHERE "); - String and = ""; - for (Map.Entry e : keys.entrySet()) - { - ColumnInfo keyCol = e.getValue(); - ParameterHolder keyColPh = createParameter(keyCol); - - sqlfWherePK.append(and); - sqlfWherePK.append("("); - sqlfWherePK.appendIdentifier(keyCol.getSelectIdentifier()); - sqlfWherePK.append(" = "); - appendParameterOrVariable(sqlfWherePK, keyColPh); - if (keyCol.isNullable()) - { - sqlfWherePK.append(" OR "); - sqlfWherePK.appendIdentifier(keyCol.getSelectIdentifier()); - sqlfWherePK.append(" IS NULL AND "); - appendParameterOrVariable(sqlfWherePK, keyColPh); - sqlfWherePK.append(" IS NULL"); - } - sqlfWherePK.append(")"); - and = " AND "; - } - return sqlfWherePK; - } - - private String sqlServerVariableDeclaration(SQLFragment sqlfDeclare, ParameterHolder ph) - { - assert(_dialect.isSqlServer()); - String variable = ph.variableName; - sqlfDeclare.append(variable); - sqlfDeclare.append(" "); - JdbcType jdbcType = ph.p.getType(); - assert null != jdbcType; - String type = _dialect.getSqlTypeName(jdbcType); - assert null != type; - - // Workaround - SQLServer doesn't support TEXT, NTEXT, or IMAGE as local variables in statements, but is OK with NVARCHAR(MAX) - if (jdbcType.isText()) - { - if ("NTEXT".equalsIgnoreCase(type) || "TEXT".equalsIgnoreCase(type) || ph.getScale()>4000) - type = "NVARCHAR(MAX)"; - else - type = "NVARCHAR(4000)"; - } - // Add scale and precision for decimal values specifying scale - else if (jdbcType.isDecimal() && ph.getScale() > 0) - { - type = type + "(" + ph.getPrecision() + "," + ph.getScale() + ")"; - } - - sqlfDeclare.append(type); - return variable; - } - - /* - * We could use SQLFragment.appendValue() for most of these. However, here it is important to force - * the use of inline literal values. SQLFragment.appendValue() does not guarantee that. - */ - private void toLiteral(SQLFragment f, Object value) - { - if (null == value) - { - f.append("NULL"); - return; - } - if (value instanceof Number) - { - f.append(value.toString()); - return; - } - if (value instanceof SQLFragment.NowTimestamp now) - { - f.appendValue(now); - return; - } - if (value instanceof java.sql.Date sqlDate) - { - f.append("{d ").append(_dialect.getStringHandler().quoteStringLiteral(DateUtil.formatIsoDate(sqlDate))).append("}"); - return; - } - else if (value instanceof java.util.Date date) - { - f.append("{ts ").append(_dialect.getStringHandler().quoteStringLiteral(DateUtil.formatIsoDateShortTime(date))).append("}"); - return; - } - assert value instanceof String; - f.append(_dialect.getStringHandler().quoteStringLiteral(String.valueOf(value))); - } - - @SuppressWarnings("JUnitMalformedDeclaration") - public static class TestCase extends Assert - { - final static String DATA_CLASS_NAME = "StatementUtilsTestDataClass"; - final static String VOCAB_DOMAIN_KIND = "Vocabulary"; // VocabularyDomainKind.KIND_NAME - final static String VOCAB_DOMAIN_NAME = "StatementUtilsVocabularyDomain"; - - final Container container; - final TableInfo dataClassTable; - final TableInfo principalsTable; - final UpdateableTableInfo testTable; - final User user; - final Set vocabParameters = CaseInsensitiveHashSet.of("Age", "AgeMVIndicator", "Color", "ColorMVIndicator"); - final Set vocabProps; - - // Flag to run tests against one or both (Postgres, SQL Server) SqlDialects. Set to false by default - // since tests are run in both environments in CI. See "otherSqlDialect". - final boolean runOtherDialect = false; - - // This is a mock SqlDialect that mocks the alternative SqlDialect configuration to the current configuration. - // So, if the tests are running in a Postgres environment, then this represents a SQL Server SqlDialect - // and vice versa. This is useful for getting code coverage across code paths for both dialects in a single - // test run. Enabled via the "runOtherDialect" flag. - final SqlDialect otherSqlDialect; - - public TestCase() throws Exception - { - container = JunitUtil.getTestContainer(); - user = TestContext.get().getUser(); - - dataClassTable = QueryService.get().getUserSchema(user, container, ExpSchema.SCHEMA_EXP_DATA).getTableOrThrow(DATA_CLASS_NAME); - principalsTable = DbSchema.get("core", DbSchemaType.Module).getTable("principals"); - testTable = DbSchema.get("test", DbSchemaType.Module).getTable("testtable2"); - - // Initialize vocab domain properties - { - var vocabDomainKind = PropertyService.get().getDomainKindByName(VOCAB_DOMAIN_KIND); - var vocabDomainURI = vocabDomainKind.generateDomainURI(null, VOCAB_DOMAIN_NAME, container, user); - var vocabDomain = PropertyService.get().getDomain(container, vocabDomainURI); - assertNotNull(vocabDomain); - vocabProps = Set.of(vocabDomain.getPropertyByName("Age"), vocabDomain.getPropertyByName("Color")); - } - - if (runOtherDialect) - { - SqlDialect defaultDialect = principalsTable.getSqlDialect(); - boolean isPostgres = defaultDialect.isPostgreSQL(); - - otherSqlDialect = new MockSqlDialect() - { - @Override - public String addReselect(SQLFragment sql, ColumnInfo column, @Nullable String proposedVariable) - { - return defaultDialect.addReselect(sql, column, proposedVariable); - } - - @Override - public String getGuidType() - { - return defaultDialect.getGuidType(); - } - - @Override - public @Nullable String getSqlTypeName(JdbcType type) - { - return defaultDialect.getSqlTypeName(type); - } - - @Override - public boolean isPostgreSQL() - { - // Returns true in SQL Server configured environments - return !isPostgres; - } - - @Override - public boolean isSqlServer() - { - // Returns true in Postgres configured environments - return isPostgres; - } - }; - } - else - { - otherSqlDialect = null; - } - } - - @BeforeClass - public static void createDomains() throws Exception - { - var container = JunitUtil.getTestContainer(); - var user = TestContext.get().getUser(); - - // Create a data class domain - ExperimentService.get().createDataClass(container, user, DATA_CLASS_NAME, null, List.of(new GWTPropertyDescriptor("aa", "int")), List.of(), null, null); - - // Create a vocabulary domain - { - GWTPropertyDescriptor prop1 = new GWTPropertyDescriptor(); - prop1.setRangeURI("int"); - prop1.setName("Age"); - prop1.setMvEnabled(true); - - GWTPropertyDescriptor prop2 = new GWTPropertyDescriptor(); - prop2.setRangeURI("string"); - prop2.setName("Color"); - - GWTDomain domain = new GWTDomain<>(); - domain.setName(VOCAB_DOMAIN_NAME); - domain.setFields(List.of(prop1, prop2)); - - DomainUtil.createDomain(VOCAB_DOMAIN_KIND, domain, null, container, user, VOCAB_DOMAIN_NAME, null, false); - } - } - - @AfterClass - public static void cleanup() - { - deleteTestContainer(); - } - - @Test - public void testToLiteral() - { - boolean isPostgres = principalsTable.getSqlDialect().isPostgreSQL(); - - var statement = new StatementUtils(Operation.insert, principalsTable); - Function runToLiteral = (value) -> { - var sql = new SQLFragment(); - statement.toLiteral(sql, value); - return sql; - }; - - var dateLong = 1749759500016L; // Thu Jun 12 2025 13:18:20 GMT-0700 (Pacific Daylight Time) - - // null value - var actual = runToLiteral.apply(null); - assertEquals(new SQLFragment("NULL"), actual); - - // Number - actual = runToLiteral.apply(1234567890); - assertEquals(new SQLFragment("1234567890"), actual); - - // NowTimestamp - var now = new SQLFragment.NowTimestamp(dateLong); - actual = runToLiteral.apply(now); - assertEquals(new SQLFragment().appendValue(now), actual); - - // sql.Date - var sqlDate = new java.sql.Date(dateLong); - var dateFormat = new SimpleDateFormat(DateUtil.getStandardDateFormatString()); - var expected = String.format(isPostgres ? "{d '%s'}" : "{d N'%s'}", dateFormat.format(sqlDate)); - - actual = runToLiteral.apply(sqlDate); - assertEquals(new SQLFragment(expected), actual); - - // util.Date - var utilDate = new java.util.Date(dateLong); - dateFormat = new SimpleDateFormat(DateUtil.getStandardDateTimeFormatString()); - expected = String.format(isPostgres ? "{ts '%s'}" : "{ts N'%s'}", dateFormat.format(utilDate)); - - actual = runToLiteral.apply(utilDate); - assertEquals(new SQLFragment(expected), actual); - } - - @Test - public void testCreateStatementValidation() throws Exception - { - try (var conn = getConnection()) - { - var nonUpdateTable = new VirtualTable<>(DbSchema.get("test", DbSchemaType.Module), "virtualInsanity", null); - - var exception = Assert.assertThrows(IllegalArgumentException.class, () -> new StatementUtils(Operation.merge, nonUpdateTable).createStatement(conn, container, user)); - assertEquals("Table must be an UpdateableTableInfo", exception.getMessage()); - - // Unreachable with current mocks -// var nonDatabaseTable = QueryService.get().getUserSchema(user, container, "core").getTableOrThrow("Principals"); -// exception = Assert.assertThrows(IllegalArgumentException.class, () -> new StatementUtils(Operation.merge, nonDatabaseTable).createStatement(conn, container, user)); -// assertEquals("Table must be a database table", exception.getMessage()); - -// exception = Assert.assertThrows(IllegalArgumentException.class, () -> { -// var noIdentifierTable = principalsTable.getMetaDataIdentifier(). -// new StatementUtils(Operation.merge, nonDatabaseTable).dialect(new MockSqlDialect()).createStatement(conn, container, user); -// }); -// assertEquals("Table must have a metadata identifier", exception.getMessage()); - - exception = Assert.assertThrows(IllegalArgumentException.class, () -> new StatementUtils(Operation.merge, principalsTable).dialect(new MockSqlDialect()).createStatement(conn, container, user)); - assertEquals("Merge is only supported/tested on postgres and sql server", exception.getMessage()); - } - } - - @Test - public void testGetKeys() - { - var containerFieldKey = FieldKey.fromParts("Container"); - var rowIdFieldKey = FieldKey.fromParts("RowId"); - var textFieldKey = FieldKey.fromParts("Text"); - - var updateTable = testTable; - var table = updateTable.getSchemaTableInfo(); - - // Pre-conditions - var pkColumnNames = new CaseInsensitiveHashSet(testTable.getPkColumnNames()); - assertEquals(2, pkColumnNames.size()); - assertTrue(pkColumnNames.contains(containerFieldKey.getName())); - assertTrue(pkColumnNames.contains(textFieldKey.getName())); - assertNotNull(testTable.getColumn(rowIdFieldKey)); - - // The "Container" column is always resolved if present on the table - var keys = StatementUtils.getKeys(updateTable, table, null, Set.of(containerFieldKey.getName()), false); - assertEquals(1, keys.size()); - assertTrue(keys.containsKey(containerFieldKey)); - - // The "Container" column is only resolved even when in the explicit name map - keys = StatementUtils.getKeys(updateTable, table, null, Set.of(containerFieldKey.getName()), false); - assertEquals(1, keys.size()); - assertTrue(keys.containsKey(containerFieldKey)); - - // The "Container" column is also resolved even when not in the explicit key column map. Other key columns are included as well. - keys = StatementUtils.getKeys(updateTable, table, null, Set.of(textFieldKey.getName()), false); - assertEquals(2, keys.size()); - assertTrue(keys.containsKey(containerFieldKey)); - assertTrue(keys.containsKey(textFieldKey)); - - // All explicitly named columns should resolve as columns on the table - var exception = Assert.assertThrows(IllegalArgumentException.class, () -> StatementUtils.getKeys(updateTable, table, null, Set.of(textFieldKey.getName(), "Beep"), false)); - assertEquals("Column not found: Beep", exception.getMessage()); - - // Furnish an explicit "objectURIColumnName" and expect it to be included when preferPKOverObjectUriAsKey = false - keys = StatementUtils.getKeys(updateTable, table, "RowId", null, false); - assertEquals(2, keys.size()); - assertTrue(keys.containsKey(containerFieldKey)); - assertTrue(keys.containsKey(rowIdFieldKey)); - - // Furnish an explicit "objectURIColumnName" and expect it to NOT be included when preferPKOverObjectUriAsKey = true - keys = StatementUtils.getKeys(updateTable, table, "RowId", null, true); - assertEquals(2, keys.size()); - assertTrue(keys.containsKey(containerFieldKey)); - assertTrue(keys.containsKey(textFieldKey)); - - keys = StatementUtils.getKeys(updateTable, table, null, null, false); - assertEquals(2, keys.size()); - assertTrue(keys.containsKey(containerFieldKey)); - assertTrue(keys.containsKey(textFieldKey)); - } - - @Test - public void testInsert() throws Exception - { - ParameterMapStatement m = null; - try (Connection conn = getConnection()) - { - m = insertStatement(conn, principalsTable, container, user, true, true); - m.close(); m = null; - - m = insertStatement(conn, testTable, container, user, true, true); - m.close(); m = null; - } - finally - { - if (null != m) - m.close(); - } - } - - @Test - public void testInsertWithExtensibleDomain() throws Exception - { - ParameterMapStatement m = null; - try (Connection conn = getConnection(dataClassTable)) - { - StatementUtils statement; - - // Insert - { - var validateInsert = new Function() - { - @Override - public Object apply(StatementUtils s) - { - boolean isPostgres = s._dialect.isPostgreSQL(); - - assertTrue(s._columnTracker.insertColumns.contains("Created")); - assertTrue(s._columnTracker.insertColumns.contains("CreatedBy")); - assertTrue(s._columnTracker.insertColumns.contains("Modified")); - assertTrue(s._columnTracker.insertColumns.contains("ModifiedBy")); - assertTrue(s._columnTracker.updateColumns.isEmpty()); - - if (isPostgres) - { - assertTrue(s._columnTracker.selectColumns.contains("_rowid_")); - assertTrue(s._columnTracker.selectColumns.contains("_$objectid$_")); - } - else - { - // Variables are not used in SQL Server - assertFalse(s._columnTracker.selectColumns.contains("_rowid_")); - assertTrue(s._columnTracker.selectColumns.contains("@_objectid_")); - } - - var parameterKeys = s.parameters.keySet(); - assertTrue(vocabParameters.stream().noneMatch(parameterKeys::contains)); - return null; - } - }; - - statement = insertStatement(dataClassTable, true, true); - m = statement.createStatement(conn, container, user); - m.close(); - m = null; - validateInsert.apply(statement); - - if (runOtherDialect) - { - statement = insertStatement(dataClassTable, true, true); - statement.dialect(otherSqlDialect); - m = statement.createStatement(conn, container, user); - m.close(); - m = null; - validateInsert.apply(statement); - } - } - - // Insert with vocabulary properties - { - statement = insertStatement(dataClassTable, true, true); - statement.setVocabularyProperties(vocabProps); - m = statement.createStatement(conn, container, user); - m.close(); - m = null; - assertTrue(statement.parameters.keySet().containsAll(vocabParameters)); - } - } - finally - { - if (null != m) - m.close(); - } - } - - @Test - public void testUpdate() throws Exception - { - ParameterMapStatement m = null; - try (Connection conn = getConnection()) - { - m = updateStatement(conn, principalsTable, container, user, true, true); - m.close(); m = null; - - m = updateStatement(conn, testTable, container, user, true, true); - m.close(); m = null; - } - finally - { - if (null != m) - m.close(); - } - } - - @Test - public void testUpdateWithExtensibleDomain() throws Exception - { - ParameterMapStatement m = null; - try (Connection conn = getConnection(dataClassTable)) - { - StatementUtils statement; - - // Update - { - var validateUpdate = new Function() - { - @Override - public Object apply(StatementUtils s) - { - assertTrue(s._columnTracker.insertColumns.isEmpty()); - assertFalse(s._columnTracker.updateColumns.contains("Created")); - assertFalse(s._columnTracker.updateColumns.contains("CreatedBy")); - assertTrue(s._columnTracker.updateColumns.contains("Modified")); - assertTrue(s._columnTracker.updateColumns.contains("ModifiedBy")); - assertTrue(s._columnTracker.selectColumns.isEmpty()); - var parameterKeys = s.parameters.keySet(); - assertTrue(vocabParameters.stream().noneMatch(parameterKeys::contains)); - return null; - } - }; - - statement = updateStatement(dataClassTable, true, true); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - validateUpdate.apply(statement); - - Set allUpdateColumns = new CaseInsensitiveHashSet(statement._columnTracker.updateColumns); - - if (runOtherDialect) - { - statement = updateStatement(dataClassTable, true, true); - statement.dialect(otherSqlDialect); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - validateUpdate.apply(statement); - } - - statement = updateStatement(dataClassTable, false, false); - statement.noupdate(CaseInsensitiveHashSet.of("Modified", "ModifiedBy")); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - assertFalse(statement._columnTracker.updateColumns.contains("Modified")); - assertFalse(statement._columnTracker.updateColumns.contains("ModifiedBy")); - - statement = updateStatement(dataClassTable, false, false); - statement.noupdate(allUpdateColumns); - m = statement.createStatement(conn, container, user); - var debugSql = m.getDebugSql(); - m.close(); m = null; - assertTrue(debugSql.contains("'noop' WHERE 1 <> 1")); - } - - // Update with vocabulary properties - { - statement = updateStatement(dataClassTable, true, true); - statement.setVocabularyProperties(vocabProps); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - assertTrue(statement.parameters.keySet().containsAll(vocabParameters)); - } - } - finally - { - if (null != m) - m.close(); - } - } - - @Test - public void testUpdateWithObjectUriColumn() throws Exception - { - // Arrange - // Create a list - String listName = "StatementUtilsTestList"; - { - // Create a list domain - var listDef = ListService.get().createList(container, listName, ListDefinition.KeyType.AutoIncrementInteger); - listDef.setKeyName("pk"); - - Domain domain = requireNonNull(listDef.getDomain()); - addProperty(domain, "pk", PropertyType.INTEGER); - addProperty(domain, "name", PropertyType.STRING); - - listDef.save(user); - } - - ParameterMapStatement m = null; - TableInfo listTable = requireNonNull(ListService.get().getList(container, listName)).getTable(user, container); - assertNotNull(listTable); - - try (Connection conn = getConnection(listTable)) - { - StatementUtils statement; - var expectedNoUpdateColumns = CaseInsensitiveHashSet.of("EntityId", "Created", "CreatedBy"); - var expectedUpdateColumns = CaseInsensitiveHashSet.of("DIImportHash", "LastIndexed", "Modified", "ModifiedBy", "Name"); - - // Update statement (selectIds = true, autoFillDefaultColumns = true) - { - statement = StatementUtils.updateStatement(listTable, true, true); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - - // Assert - assertEquals(expectedNoUpdateColumns, statement._dontUpdateColumnNames); - assertTrue(statement._columnTracker.insertColumns.isEmpty()); - assertTrue(statement._columnTracker.selectColumns.isEmpty()); - - if (runOtherDialect) - { - statement = StatementUtils.updateStatement(listTable, true, true); - statement.dialect(otherSqlDialect); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - } - - assertEquals(expectedNoUpdateColumns, statement._dontUpdateColumnNames); - assertTrue(statement._columnTracker.insertColumns.isEmpty()); - assertTrue(statement._columnTracker.selectColumns.isEmpty()); - - assertEquals(expectedUpdateColumns, statement._columnTracker.updateColumns); - } - - // Update statement (selectIds = false, autoFillDefaultColumns = false) - { - statement = StatementUtils.updateStatement(listTable, false, false); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - - // Assert - assertEquals(expectedNoUpdateColumns, statement._dontUpdateColumnNames); - assertTrue(statement._columnTracker.insertColumns.isEmpty()); - assertEquals(expectedUpdateColumns, statement._columnTracker.updateColumns); - - if (runOtherDialect) - { - statement = StatementUtils.updateStatement(listTable, false, false); - statement.dialect(otherSqlDialect); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - } - - assertEquals(expectedNoUpdateColumns, statement._dontUpdateColumnNames); - assertTrue(statement._columnTracker.insertColumns.isEmpty()); - assertEquals(expectedUpdateColumns, statement._columnTracker.updateColumns); - } - } - finally - { - if (null != m) - m.close(); - } - } - - @Test - public void testMerge() throws Exception - { - ParameterMapStatement m = null; - try (Connection conn = getConnection()) - { - m = mergeStatement(conn, principalsTable, null, null, null, container, user, false, true, false); - m.close(); m = null; - - if (runOtherDialect) - { - StatementUtils statement = mergeStatement(principalsTable, null, null, null, false, true, false); - statement.dialect(otherSqlDialect); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - } - - m = mergeStatement(conn, testTable, null, null, null, container, user, false, true, false); - m.close(); m = null; - - if (runOtherDialect) - { - StatementUtils statement = mergeStatement(testTable, null, null, null, false, true, false); - statement.dialect(otherSqlDialect); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - } - } - finally - { - if (null != m) - m.close(); - } - } - - @Test - public void testMergeWithExtensibleDomain() throws Exception - { - ParameterMapStatement m = null; - try (Connection conn = getConnection(dataClassTable)) - { - StatementUtils statement; - - // Merge - { - var validateMerge = new Function() - { - @Override - public Object apply(StatementUtils s) - { - boolean isPostgres = s._dialect.isPostgreSQL(); - - assertTrue(s._columnTracker.insertColumns.contains("Container")); - assertTrue(s._columnTracker.insertColumns.contains("LSID")); - assertFalse(s._columnTracker.updateColumns.contains("Container")); - assertFalse(s._columnTracker.updateColumns.contains("LSID")); - - if (isPostgres) - { - assertTrue(s._columnTracker.selectColumns.contains("_rowid_")); - assertTrue(s._columnTracker.selectColumns.contains("_$objectid$_")); - } - else - { - // Variables are not used in SQL Server - assertFalse(s._columnTracker.selectColumns.contains("_rowid_")); - assertTrue(s._columnTracker.selectColumns.contains("@_objectid_")); - } - - var parameterKeys = s.parameters.keySet(); - assertTrue(vocabParameters.stream().noneMatch(parameterKeys::contains)); - return null; - } - }; - - statement = mergeStatement(dataClassTable, null, null, null, true, true, false); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - validateMerge.apply(statement); - - var updateColumns = new CaseInsensitiveHashSet(statement._columnTracker.updateColumns); - - if (runOtherDialect) - { - statement = mergeStatement(dataClassTable, null, null, null, true, true, false); - statement.dialect(otherSqlDialect); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - validateMerge.apply(statement); - } - - // TODO: This generates a SQL parsing error in Postgres due to the reselect statement coming before the WHERE clause -// statement = mergeStatement(dataClassTable, null, CaseInsensitiveHashSet.of("RunId"), updateColumns, true, true, false); -// m = statement.createStatement(conn, container, user); -// m.close(); m = null; -// assertTrue(statement._columnTracker.updateColumns.isEmpty()); - } - - // Merge with vocabulary properties - { - statement = mergeStatement(dataClassTable, null, null, null, false, true, false); - statement.setVocabularyProperties(vocabProps); - m = statement.createStatement(conn, container, user); - m.close(); m = null; - assertTrue(statement.parameters.keySet().containsAll(vocabParameters)); - } - } - finally - { - if (null != m) - m.close(); - } - } - - private static void addProperty(Domain d, String name, PropertyType pt) - { - DomainProperty p = d.addProperty(); - p.setName(name); - p.setPropertyURI(d.getTypeURI() + "#" + name); - p.setRangeURI(pt.getTypeUri()); - } - - private Connection getConnection() throws SQLException - { - return getConnection(principalsTable); - } - - private Connection getConnection(TableInfo table) throws SQLException - { - return table.getSchema().getScope().getConnection(); - } - } -} +/* + * Copyright (c) 2012-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.data; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.CaseInsensitiveMapWrapper; +import org.labkey.api.collections.Sets; +import org.labkey.api.data.dialect.MockSqlDialect; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.dataiterator.SimpleTranslator; +import org.labkey.api.dataiterator.TableInsertUpdateDataIterator; +import org.labkey.api.exp.MvColumn; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.list.ListDefinition; +import org.labkey.api.exp.list.ListService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainKind; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.DomainUtil; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.gwt.client.model.GWTDomain; +import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; +import org.labkey.api.query.AliasManager; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.security.User; +import org.labkey.api.util.DateUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JunitUtil; +import org.labkey.api.util.TestContext; +import org.labkey.api.util.logging.LogHelper; + +import java.sql.Connection; +import java.sql.SQLException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Stream; + +import static java.util.Objects.requireNonNull; +import static org.labkey.api.util.JunitUtil.deleteTestContainer; + +public class StatementUtils +{ + private static final Logger _log = LogHelper.getLogger(StatementUtils.class, "SQL insert/update/delete generation"); + + public enum Operation {insert, update, merge} + + // configuration parameters + private Operation _operation; + private SqlDialect _dialect; + private final TableInfo _targetTable; + private Set _keyColumnNames = null; // override the primary key of _table + private Set _skipColumnNames = Set.of(); + private final Set _dontUpdateColumnNames = new CaseInsensitiveHashSet(); + private boolean _updateBuiltInColumns = false; // default to false, this should usually be handled by StandardDataIteratorBuilder + private boolean _selectIds = false; + private boolean _selectObjectUri = false; + private boolean _allowUpdateAutoIncrement = false; + private boolean _preferPKOverObjectUriAsKey = false; + + // variable/parameter tracking helpers + private boolean useVariables = false; + private final Map _constants = new CaseInsensitiveHashMap<>(); + final Map parameters = new CaseInsensitiveMapWrapper<>(new LinkedHashMap<>()); + + // ColumnTracker is used for test instrumentation and contains sets of column names that were included in + // given operations (insert, update, select) after running createStatement(). + // This is intended for test instrumentation use only. + private record ColumnTracker(Set insertColumns, Set updateColumns, Set selectColumns) + { + public ColumnTracker() + { + this(new CaseInsensitiveHashSet(), new CaseInsensitiveHashSet(), new CaseInsensitiveHashSet()); + } + } + + private ColumnTracker _columnTracker; + + // + // builder style methods + // + + //Vocabulary adhoc properties + private Set _vocabularyProperties = new HashSet<>(); + + public StatementUtils(@NotNull Operation op, @NotNull TableInfo table) + { + _operation = op; + _dialect = table.getSqlDialect(); + _targetTable = table; + } + + public StatementUtils dialect(SqlDialect dialect) + { + _dialect = dialect; + return this; + } + + public StatementUtils operation(@NotNull Operation op) + { + _operation = op; + return this; + } + + public StatementUtils constants(@NotNull Map constants) + { + _constants.putAll(constants); + return this; + } + + public StatementUtils keys(Set keyNames) + { + _keyColumnNames = keyNames; + return this; + } + + public StatementUtils skip(Set skip) + { + _skipColumnNames = null==skip ? Set.of() : skip; + return this; + } + + public StatementUtils noupdate(Set noupdate) + { + if (null != noupdate) + _dontUpdateColumnNames.addAll(noupdate); + return this; + } + + public StatementUtils updateBuiltinColumns(boolean b) + { + _updateBuiltInColumns = b; + return this; + } + + public StatementUtils selectIds(boolean b) + { + _selectIds = b; + return this; + } + + public StatementUtils selectObjectUri(boolean b) + { + _selectObjectUri = b; + return this; + } + + public StatementUtils allowSetAutoIncrement(boolean b) + { + _allowUpdateAutoIncrement = b; + return this; + } + + public StatementUtils setVocabularyProperties(Set vocabularyProperties) + { + _vocabularyProperties = vocabularyProperties; + return this; + } + + public StatementUtils setPreferPKOverObjectUriAsKey(boolean preferPKOverObjectUriAsKey) + { + _preferPKOverObjectUriAsKey = preferPKOverObjectUriAsKey; + return this; + } + + private static StatementUtils insertStatement(TableInfo table, boolean selectIds, boolean autoFillDefaultColumns) + { + return new StatementUtils(Operation.insert, table) + .updateBuiltinColumns(autoFillDefaultColumns) + .selectIds(selectIds); + } + + /** + * Create a reusable SQL Statement for inserting rows into a labkey relationship. The relationship + * persisted directly in the database (SchemaTableInfo), or via the OntologyManager tables. + *

+ * QueryService shouldn't really know about the internals of exp.Object and exp.ObjectProperty etc. + * However, I can only keep so many levels of abstraction in my head at once. + *

+ * NOTE: this is currently fairly expensive for updating one row into an Ontology stored relationship on Postgres. + * This shouldn't be a big problem since we don't usually need to optimize the one-row case, and we're moving + * to provisioned tables for major datatypes. + */ + public static ParameterMapStatement insertStatement(Connection conn, TableInfo table, @Nullable Container c, @Nullable User user, boolean selectIds, boolean autoFillDefaultColumns) throws SQLException + { + return insertStatement(table, selectIds, autoFillDefaultColumns) + .createStatement(conn, c, user); + } + + private static StatementUtils updateStatement(TableInfo table, boolean selectIds, boolean autoFillDefaultColumns) + { + return new StatementUtils(Operation.update, table) + .updateBuiltinColumns(autoFillDefaultColumns) + .selectIds(selectIds); + } + + /** + * Create a reusable SQL Statement for updating rows into a labkey relationship. The relationship + * persisted directly in the database (SchemaTableInfo), or via the OntologyManager tables. + *

+ * QueryService shouldn't really know about the internals of exp.Object and exp.ObjectProperty etc. + * However, I can only keep so many levels of abstraction in my head at once. + *

+ * NOTE: this is currently fairly expensive for updating one row into an Ontology stored relationship on Postgres. + * This shouldn't be a big problem since we don't usually need to optimize the one-row case, and we're moving + * to provisioned tables for major datatypes. + */ + public static ParameterMapStatement updateStatement(Connection conn, TableInfo table, @Nullable Container c, User user, boolean selectIds, boolean autoFillDefaultColumns) throws SQLException + { + return updateStatement(table, selectIds, autoFillDefaultColumns) + .createStatement(conn, c, user); + } + + private static StatementUtils mergeStatement(TableInfo table, @Nullable Set keyNames, @Nullable Set skipColumnNames, @Nullable Set dontUpdate, boolean selectIds, boolean autoFillDefaultColumns, boolean supportsAutoIncrementKey) + { + return new StatementUtils(Operation.merge, table) + .keys(keyNames) + .skip(skipColumnNames) + .allowSetAutoIncrement(supportsAutoIncrementKey) + .noupdate(dontUpdate) + .updateBuiltinColumns(autoFillDefaultColumns) + .selectIds(selectIds); + } + + public static ParameterMapStatement mergeStatement(Connection conn, TableInfo table, @Nullable Set keyNames, @Nullable Set skipColumnNames, @Nullable Set dontUpdate, @Nullable Container c, @Nullable User user, boolean selectIds, boolean autoFillDefaultColumns, boolean supportsAutoIncrementKey) throws SQLException + { + return mergeStatement(table, keyNames, skipColumnNames, dontUpdate, selectIds, autoFillDefaultColumns, supportsAutoIncrementKey) + .createStatement(conn, c, user); + } + + /* + * Parameter and Variable helpers + */ + + private static class ParameterHolder + { + ParameterHolder(Parameter p) + { + this.p = p; + _columnInfo = null; + } + + ParameterHolder(Parameter p, ColumnInfo c) + { + this.p = p; + _columnInfo = c; + } + + int getScale() + { + var type = requireNonNull(p.getType()); + if (null == _columnInfo || _columnInfo.getScale() <= 0) + return -1; + // GUID.isText()==true + if (JdbcType.GUID != type && (type.isText() || type.isDecimal())) + return _columnInfo.getScale(); + return -1; + } + + int getPrecision() + { + return null==_columnInfo ? -1 : _columnInfo.getPrecision(); + } + + final Parameter p; + final ColumnInfo _columnInfo; + String variableName = null; + Object constantValue = null; + boolean isConstant = false; + } + + private final static String pgRowVarPrefix = "$1."; + private String makeVariableName(String name) + { + String shortName = StringUtils.substring(name,0,32); // name is just for readability, make it short + String uniquePrefix = (_dialect.isSqlServer() ? "@" : pgRowVarPrefix) + ("p" + (parameters.size()+1) + "_"); + return uniquePrefix + AliasManager.makeLegalName(shortName, _dialect, true, uniquePrefix.length()); + } + + private String makePgRowTypeName(String variableName) + { + return StringUtils.substringAfter(variableName, pgRowVarPrefix); + } + + private ParameterHolder createParameter(ColumnInfo c) + { + ParameterHolder ph = parameters.get(c.getName()); + if (null == ph) + { + ph = new ParameterHolder(new Parameter(c.getName(), c.getPropertyURI(), null, c.getJdbcType()), c); + // NOTE: earlier DataIterator should probably split file into two columns: attachment_name, attachment_body + if (c.getInputType().equalsIgnoreCase("file") && c.getJdbcType() == JdbcType.VARCHAR) + ph.p.setFileAsName(true); + initParameterHolder(ph); + parameters.put(c.getName(), ph); + } + return ph; + } + + private void initParameterHolder(ParameterHolder ph) + { + String name = ph.p.getName(); + JdbcType type = ph.p.getType(); + assert null != type; + if (_constants.containsKey(name)) + { + Object value = Parameter.getValueToBind(_constants.get(name), type); + if (null == value || value instanceof Number || value instanceof String || value instanceof java.util.Date) + { + ph.isConstant = true; + ph.constantValue = value; + } + } + ph.variableName = makeVariableName(name); + } + + private ParameterHolder createParameter(String name, JdbcType type) + { + ParameterHolder ph = parameters.get(name); + if (null == ph) + { + ph = new ParameterHolder(new Parameter(name, type)); + initParameterHolder(ph); + parameters.put(name, ph); + } + return ph; + } + + private ParameterHolder createParameter(String name, String uri, JdbcType type) + { + ParameterHolder ph = parameters.get(name); + if (null == ph) + { + ph = new ParameterHolder(new Parameter(name, uri, null, type)); + initParameterHolder(ph); + parameters.put(name, ph); + } + return ph; + } + + private SQLFragment appendParameterOrVariable(SQLFragment f, ParameterHolder ph) + { + if (ph.isConstant) + { + toLiteral(f, ph.constantValue); + } + else if (useVariables) + { + f.append(ph.variableName); + } + else + { + f.append("?"); + f.add(ph.p); + } + return f; + } + + private SQLFragment appendPropertyValue(SQLFragment f, DomainProperty dp, ParameterHolder p) + { + if (dp.getJdbcType() == JdbcType.BOOLEAN) + { + f.append("CASE CAST("); + appendParameterOrVariable(f, p); + f.append(" AS ").append(_dialect.getBooleanDataType()).append(")") + .append(" WHEN ").append(_dialect.getBooleanTRUE()).append(" THEN 1.0 ") + .append(" WHEN ").append(_dialect.getBooleanFALSE()).append(" THEN 0.0 ") + .append(" ELSE NULL END"); + return f; + } + else + { + return appendParameterOrVariable(f, p); + } + } + + private void appendSQLFObjectProperty(SQLFragment sqlfObjectProperty, DomainProperty dp, String objectIdVar, String ifTHEN, String ifEND) + { + PropertyType propertyType = dp.getPropertyDescriptor().getPropertyType(); + ParameterHolder v = createParameter(dp.getName(), dp.getPropertyURI(), propertyType.getJdbcType()); + ParameterHolder mv = createParameter(dp.getName()+ MvColumn.MV_INDICATOR_SUFFIX, dp.getPropertyURI() + MvColumn.MV_INDICATOR_SUFFIX, JdbcType.VARCHAR); + sqlfObjectProperty.append("IF ("); + appendPropertyValue(sqlfObjectProperty, dp, v); + sqlfObjectProperty.append(" IS NOT NULL"); + if (dp.isMvEnabled()) + { + sqlfObjectProperty.append(" OR "); + appendParameterOrVariable(sqlfObjectProperty, mv); + sqlfObjectProperty.append(" IS NOT NULL"); + } + sqlfObjectProperty.append(")"); + sqlfObjectProperty.append(ifTHEN); + sqlfObjectProperty.append("INSERT INTO exp.ObjectProperty (objectid, propertyid, typetag, mvindicator, "); + sqlfObjectProperty.append(propertyType.getValueTypeColumn()); + sqlfObjectProperty.append(") VALUES ("); + sqlfObjectProperty.append(objectIdVar); + sqlfObjectProperty.append(",").appendValue(dp.getPropertyId()); + sqlfObjectProperty.append(",").appendStringLiteral(String.valueOf(propertyType.getStorageType()), _dialect); + sqlfObjectProperty.append(","); + if (dp.isMvEnabled()) + appendParameterOrVariable(sqlfObjectProperty, mv); + else + sqlfObjectProperty.append("NULL"); + sqlfObjectProperty.append(","); + appendPropertyValue(sqlfObjectProperty, dp, v); + sqlfObjectProperty.append(")").appendEOS(); + sqlfObjectProperty.append(ifEND); + sqlfObjectProperty.appendEOS(); + } + + private void appendSQLFDeleteObjectProperty(SQLFragment sqlfDelete, String objectIdVar, List domainProperties, Set vocabularyProperties) + { + var properties = null == domainProperties ? vocabularyProperties : domainProperties; + sqlfDelete.append("DELETE FROM exp.ObjectProperty WHERE ObjectId = "); + sqlfDelete.append(objectIdVar); + sqlfDelete.append(" AND PropertyId IN ("); + String separator = ""; + for (DomainProperty property : properties) + { + sqlfDelete.append(separator); + separator = ", "; + sqlfDelete.appendValue(property.getPropertyId()); + } + sqlfDelete.append(")").appendEOS(); + } + + private void setObjectUriPreselect(SQLFragment sqlfPreselectObject, TableInfo table, LinkedHashMap keys, String objectURIVar, String objectURIColumnName, ParameterHolder objecturiParameter) + { + String setKeyword = _dialect.isPostgreSQL() ? "" : "SET "; + if (Operation.merge == _operation || Operation.update == _operation) + { + // this seems overkill actually, but I'm focused on optimizing insert right now (MAB) + sqlfPreselectObject.append(setKeyword).append(objectURIVar).append(" = COALESCE(("); + sqlfPreselectObject.append("SELECT ").appendIdentifier(table.getColumn(objectURIColumnName).getSelectIdentifier()); + sqlfPreselectObject.append(" FROM ").append(table.getSQLName()); + sqlfPreselectObject.append(getPkWhereClause(keys)); + sqlfPreselectObject.append("),"); + appendParameterOrVariable(sqlfPreselectObject, objecturiParameter); + sqlfPreselectObject.append(")").appendEOS(); + + } + else + { + sqlfPreselectObject.append(setKeyword).append(objectURIVar).append(" = "); + appendParameterOrVariable(sqlfPreselectObject, objecturiParameter); + sqlfPreselectObject.appendEOS(); + } + } + + public ParameterMapStatement createStatement(Connection conn, @Nullable Container c, User user) throws SQLException + { + ParameterMapStatement statement = null; + try + { + statement = createStatement(conn, c, user, false); + } + catch (TableInsertUpdateDataIterator.NoUpdatableColumnInDataException e) + { + // ignore error + } + return statement; + } + + public ParameterMapStatement createStatement(Connection conn, @Nullable Container c, User user, boolean checkUpdatableColumns) throws SQLException, TableInsertUpdateDataIterator.NoUpdatableColumnInDataException + { + if (!(_targetTable instanceof UpdateableTableInfo updatable)) + throw new IllegalArgumentException("Table must be an UpdateableTableInfo"); + + TableInfo table = updatable.getSchemaTableInfo(); + + if (table.getTableType() != DatabaseTableType.TABLE) + throw new IllegalArgumentException("Table must be a database table"); + if (null == table.getMetaDataIdentifier()) + throw new IllegalArgumentException("Table must have a metadata identifier"); + + if (Operation.merge == _operation) + { + if (!_dialect.isPostgreSQL() && !_dialect.isSqlServer()) + throw new IllegalArgumentException("Merge is only supported/tested on postgres and sql server"); + } + + useVariables = Operation.merge == _operation; + String ifTHEN = _dialect.isSqlServer() ? " BEGIN " : " THEN "; + String ifEND = _dialect.isSqlServer() ? " END " : " END IF "; + + if (null != c) + { + assert null == _constants.get("container") || c.getId().equals(_constants.get("container")); + if (null == _constants.get("container")) + _constants.put("container", c.getId()); + } + + String objectURIColumnName = updatable.getObjectUriType() == UpdateableTableInfo.ObjectUriType.schemaColumn + ? updatable.getObjectURIColumnName() + : "objecturi"; + ParameterHolder objecturiParameter = null; + if (null != objectURIColumnName) + objecturiParameter = createParameter(objectURIColumnName, JdbcType.VARCHAR); + + // + // Keys for UPDATE or MERGE + // + LinkedHashMap keys = getKeys(updatable, table, objectURIColumnName, _keyColumnNames, _preferPKOverObjectUriAsKey); + + // + // exp.Objects INSERT + // + + SQLFragment sqlfDeclare = new SQLFragment(); + SQLFragment sqlfPreselectObject = new SQLFragment(); + SQLFragment sqlfInsertObject = new SQLFragment(); + SQLFragment sqlfSelectObject = new SQLFragment(); + SQLFragment sqlfObjectProperty = new SQLFragment(); + SQLFragment sqlfDelete = new SQLFragment(); + + Domain domain = updatable.getDomain(); + DomainKind domainKind = updatable.getDomainKind(); + List properties = Collections.emptyList(); + + boolean hasObjectURIColumn = objectURIColumnName != null && table.getColumn(objectURIColumnName) != null; + boolean alwaysInsertExpObject = (hasObjectURIColumn && updatable.isAlwaysInsertExpObject()) && Operation.update != _operation; + if (hasObjectURIColumn) + _dontUpdateColumnNames.add(objectURIColumnName); +// TODO Should we add created and createdby? Or make the caller decide? + if (Operation.update == _operation) + { + _dontUpdateColumnNames.add("Created"); + _dontUpdateColumnNames.add("CreatedBy"); + } + + String objectIdVar = null; + String objectURIVar = null; + boolean objectUriPreselectSet = false; + boolean isMaterializedDomain = null != domain && null != domainKind && StringUtils.isNotEmpty(domainKind.getStorageSchemaName()); + if (alwaysInsertExpObject || (null != domain && !isMaterializedDomain) || !_vocabularyProperties.isEmpty()) + { + properties = (null==domain||isMaterializedDomain) ? Collections.emptyList() : domain.getProperties(); + + if (alwaysInsertExpObject || !properties.isEmpty() || !_vocabularyProperties.isEmpty()) + { + if (!_dialect.isPostgreSQL() && !_dialect.isSqlServer()) + throw new IllegalStateException("Domains are only supported for sql server and postgres"); + + objectIdVar = _dialect.isPostgreSQL() ? "_$objectid$_" : "@_objectid_"; + sqlfDeclare.append("DECLARE ").append(objectIdVar).append(" BIGINT").appendEOS(); + objectURIVar = _dialect.isPostgreSQL() ? "_$objecturi$_" : "@_objecturi_"; + sqlfDeclare.append("DECLARE ").append(objectURIVar).append(" ").append(_dialect.getSqlTypeName(JdbcType.VARCHAR)).append("(300)").appendEOS(); + useVariables |= _dialect.isPostgreSQL(); + + ParameterHolder containerParameter = createParameter("container", JdbcType.GUID); + + // Insert a new row in exp.Object if there isn't already a row for this object + + // Grab the object's ObjectId based on the pk of the base table + if (hasObjectURIColumn || !_vocabularyProperties.isEmpty()) + { + setObjectUriPreselect(sqlfPreselectObject, table, keys, objectURIVar, objectURIColumnName, objecturiParameter); + objectUriPreselectSet = true; + } + + SQLFragment sqlfWhereObjectURI = new SQLFragment(); + sqlfWhereObjectURI.append("(ObjectURI = ").append(objectURIVar).append(")"); + + // In the update case, it's still possible that there isn't a row in exp.Object - there might have been + // no properties in the domain when the row was originally inserted + sqlfInsertObject.append("INSERT INTO exp.Object (container, objecturi, ownerobjectid) "); + sqlfInsertObject.append("SELECT "); + appendParameterOrVariable(sqlfInsertObject, containerParameter); + sqlfInsertObject.append(" AS Container,"); + appendParameterOrVariable(sqlfInsertObject, objecturiParameter); + sqlfInsertObject.append(" AS ObjectURI, "); + Long ownerObjectId = updatable.getOwnerObjectId(); + sqlfInsertObject.append( null == ownerObjectId ? "NULL" : String.valueOf(ownerObjectId) ).append(" AS OwnerObjectId"); + sqlfInsertObject.append(" WHERE NOT EXISTS (SELECT ObjectURI FROM exp.Object WHERE Container = "); + appendParameterOrVariable(sqlfInsertObject, containerParameter); + sqlfInsertObject.append(" AND ").append(sqlfWhereObjectURI).append(")").appendEOS(); + + // re-grab the object's ObjectId, in case it was just inserted + sqlfSelectObject.append(_dialect.isPostgreSQL() ? "" : "SET ").append(objectIdVar).append(" = ("); + sqlfSelectObject.append("SELECT ObjectId FROM exp.Object WHERE Container = "); + appendParameterOrVariable(sqlfSelectObject, containerParameter); + sqlfSelectObject.append(" AND ").append(sqlfWhereObjectURI).append(")").appendEOS(); + + if (Operation.insert != _operation && (!properties.isEmpty() || !_vocabularyProperties.isEmpty())) + { + // Clear out any existing property values for this domain + if (!properties.isEmpty()) + { + appendSQLFDeleteObjectProperty(sqlfDelete, objectIdVar, properties, null); + } + + // Clear out any existing ad hoc property + if (!_vocabularyProperties.isEmpty()) + { + appendSQLFDeleteObjectProperty(sqlfDelete, objectIdVar, null, _vocabularyProperties); + } + } + } + } + + if (_selectObjectUri) + { + if (objectURIVar == null) + { + objectURIVar = _dialect.isPostgreSQL() ? "_$objecturi$_" : "@_objecturi_"; + sqlfDeclare.append("DECLARE ").append(objectURIVar).append(" ").append(_dialect.getSqlTypeName(JdbcType.VARCHAR)).append("(300)").appendEOS(); + } + + if (!objectUriPreselectSet && (hasObjectURIColumn || !_vocabularyProperties.isEmpty())) + { + setObjectUriPreselect(sqlfPreselectObject, table, keys, objectURIVar, objectURIColumnName, objecturiParameter); + } + } + + + // + // BASE TABLE INSERT() + // + + ColumnInfo col; + List cols = new ArrayList<>(); + List values = new ArrayList<>(); + Set done = Sets.newCaseInsensitiveHashSet(); + + if (_updateBuiltInColumns && Operation.update != _operation) + { + col = table.getColumn("Owner"); + if (null != col && null != user) + { + cols.add(col); + values.add(new SQLFragment().appendValue(user.getUserId())); + done.add("Owner"); + } + col = table.getColumn("CreatedBy"); + if (null != col && null != user) + { + cols.add(col); + values.add(new SQLFragment().appendValue(user.getUserId())); + done.add("CreatedBy"); + } + col = table.getColumn("Created"); + if (null != col) + { + cols.add(col); + values.add(new SQLFragment().appendValue(new SQLFragment.NowTimestamp())); + done.add("Created"); + } + } + + ColumnInfo colModifiedBy = table.getColumn("ModifiedBy"); + if (_updateBuiltInColumns && null != colModifiedBy && null != user) + { + cols.add(colModifiedBy); + values.add(new SQLFragment().appendValue(user.getUserId())); + done.add("ModifiedBy"); + } + + ColumnInfo colModified = table.getColumn("Modified"); + if (_updateBuiltInColumns && null != colModified) + { + cols.add(colModified); + values.add(new SQLFragment().appendValue(new SQLFragment.NowTimestamp())); + done.add("Modified"); + } + ColumnInfo colVersion = table.getVersionColumn(); + if (_updateBuiltInColumns && null != colVersion && !done.contains(colVersion.getName())) + { + SQLFragment expr = colVersion.getVersionUpdateExpression(); + if (null != expr) + { + cols.add(colVersion); + values.add(expr); + done.add(colVersion.getName()); + } + } + + String objectIdColumnName = StringUtils.trimToNull(updatable.getObjectIdColumnName()); + ColumnInfo autoIncrementColumn = null; + CaseInsensitiveHashMap remap = updatable.remapSchemaColumns(); + if (null == remap) + remap = CaseInsensitiveHashMap.of(); + + for (ColumnInfo column : table.getColumns()) + { + if (column instanceof WrappedColumn || column.isCalculated()) + continue; + // if we're allowing the caller to set the auto-increment column, then treat like a regular column + if (column.isAutoIncrement() && !_allowUpdateAutoIncrement) + { + autoIncrementColumn = column; + continue; + } + if (column.isVersionColumn() && column != colModified) + continue; + String name = column.getName(); + if (done.contains(name)) + continue; + done.add(name); + ColumnInfo updatableColumn = updatable.getColumn(column.getName()); + if (updatableColumn != null && updatableColumn.hasDbSequence()) + _dontUpdateColumnNames.add(column.getName()); + + SQLFragment valueSQL = new SQLFragment(); + if (column.getName().equalsIgnoreCase(objectIdColumnName)) + { + valueSQL.append(objectIdVar); + } + else if (column.getName().equalsIgnoreCase(updatable.getObjectURIColumnName()) && null != objecturiParameter) + { + appendParameterOrVariable(valueSQL, objecturiParameter); + } + else + { + if (null != _skipColumnNames && _skipColumnNames.contains(Objects.toString(remap.get(name),name))) + continue; + ParameterHolder ph = createParameter(column); + appendParameterOrVariable(valueSQL, ph); + } + cols.add(column); + values.add(valueSQL); + } + + boolean selectAutoIncrement = false; + + assert cols.size() == values.size() : cols.size() + " columns and " + values.size() + " values - should match"; + + // + // INSERT + // + + String comma; + String rowIdVar = null; + SQLFragment sqlfInsertInto = new SQLFragment(); + + // Construct a new column tracker for test instrumentation + _columnTracker = new ColumnTracker(); + + if (Operation.insert == _operation || Operation.merge == _operation) + { + // Create a standard INSERT INTO table (col1, col2) VALUES (val1, val2) statement + // or (for degenerate, empty values case) INSERT INTO table VALUES (DEFAULT) + sqlfInsertInto.append("INSERT INTO ").append(table.getSQLName()); + + if (values.isEmpty()) + { + sqlfInsertInto.append("\nVALUES (DEFAULT)"); + } + else + { + sqlfInsertInto.append(" ("); + comma = ""; + for (ColumnInfo colInfo : cols) + { + sqlfInsertInto.append(comma); + comma = ", "; + sqlfInsertInto.appendIdentifier(colInfo.getSelectIdentifier()); + _columnTracker.insertColumns.add(colInfo.getName()); + } + sqlfInsertInto.append(")"); + + sqlfInsertInto.append("\nSELECT "); + comma = ""; + for (SQLFragment valueSQL : values) + { + sqlfInsertInto.append(comma); + comma = ", "; + sqlfInsertInto.append(valueSQL); + } + } + + if (_selectIds && null != autoIncrementColumn) + { + selectAutoIncrement = true; + if (useVariables) + rowIdVar = "_rowid_"; + rowIdVar = _dialect.addReselect(sqlfInsertInto, autoIncrementColumn, rowIdVar); + if (useVariables) + sqlfDeclare.append("DECLARE ").append(rowIdVar).append(" BIGINT").appendEOS(); // CONSIDER: Move this into addReselect()? + } + + if (_selectObjectUri && hasObjectURIColumn) + { + _dialect.addReselect(sqlfInsertInto, table.getColumn(objectURIColumnName), objectURIVar); + } + } + + // + // UPDATE + // + + SQLFragment sqlfUpdate = new SQLFragment(); + if (Operation.update == _operation || Operation.merge == _operation) + { + // Create a standard UPDATE table SET col1 = val1, col2 = val2 statement + sqlfUpdate.append("UPDATE ").append(table.getSQLName()).append("\nSET "); + comma = ""; + int updateCount = 0; + for (int i = 0; i < cols.size(); i++) + { + col = cols.get(i); + FieldKey fk = col.getFieldKey(); + if (keys.containsKey(fk)) + continue; + + // Issue 52666: Check column remapping when looking for columns to not update + String colName = col.getName(); + if (_dontUpdateColumnNames.contains(colName) || (remap.containsKey(colName) && _dontUpdateColumnNames.contains(remap.get(colName)))) + continue; + + sqlfUpdate.append(comma); + comma = ", "; + sqlfUpdate.appendIdentifier(col.getSelectIdentifier()); + sqlfUpdate.append(" = "); + sqlfUpdate.append(values.get(i)); + _columnTracker.updateColumns.add(col.getName()); + updateCount++; + } + + if (Operation.update == _operation && updateCount == 0) + { + if (checkUpdatableColumns) + throw new TableInsertUpdateDataIterator.NoUpdatableColumnInDataException(table.getName()); + + sqlfUpdate.appendIdentifier(keys.values().iterator().next().getSelectIdentifier()); + sqlfUpdate.append(" = 'noop' WHERE 1 <> 1").appendEOS(); + } + else + { + sqlfUpdate.append(getPkWhereClause(keys)); + sqlfUpdate.appendEOS(); + } + + if (Operation.merge == _operation) + { + // updateCount can equal 0. This happens particularly when inserting into junction tables where + // there are two columns and both are in the primary key + if (0 == updateCount) + { + sqlfUpdate = new SQLFragment(); + sqlfInsertInto.append("\nWHERE NOT EXISTS (SELECT * FROM ").append(table.getSQLName()); + sqlfInsertInto.append(getPkWhereClause(keys)); + sqlfInsertInto.append(")"); + } + else + { + sqlfUpdate.append("IF "); + sqlfUpdate.append(_dialect.isSqlServer() ? "@@ROWCOUNT=0" : "NOT FOUND"); + sqlfUpdate.append(ifTHEN).append("\n\t"); + + sqlfInsertInto.appendEOS(); + sqlfInsertInto.append(ifEND); + } + } + } + + if (Operation.insert == _operation || Operation.merge == _operation) + sqlfInsertInto.appendEOS(); + + SQLFragment sqlfSelectIds = null; + + if ((_selectIds && (null != objectIdVar || null != rowIdVar)) || (_selectObjectUri && null != objectURIVar)) + { + sqlfSelectIds = new SQLFragment("SELECT "); + comma = ""; + if (_selectIds) + { + if (null != rowIdVar) + { + sqlfSelectIds.append(rowIdVar); + _columnTracker.selectColumns.add(rowIdVar); + comma = ","; + } + if (null != objectIdVar) + { + sqlfSelectIds.append(comma).append(objectIdVar); + _columnTracker.selectColumns.add(objectIdVar); + comma = ","; + } + } + + if (_selectObjectUri && null != objectURIVar) + { + sqlfSelectIds.append(comma).append(objectURIVar); + _columnTracker.selectColumns.add(objectIdVar); + } + } + + // + // ObjectProperty + // + + if (!properties.isEmpty()) + { + Set skip = updatable.skipProperties(); + if (null != skip) + done.addAll(skip); + + for (DomainProperty dp : properties) + { + // ignore property that 'wraps' a hard column + if (done.contains(dp.getName())) + continue; + appendSQLFObjectProperty(sqlfObjectProperty, dp, objectIdVar, ifTHEN, ifEND); + } + } + + if (!_vocabularyProperties.isEmpty()) + { + for (DomainProperty vocProp: _vocabularyProperties) + { + appendSQLFObjectProperty(sqlfObjectProperty, vocProp, objectIdVar, ifTHEN, ifEND); + } + } + + // + // PREPARE + // + + ParameterMapStatement ret; + + if (!useVariables) + { + SQLFragment script = new SQLFragment(); + Stream.of(sqlfDeclare, sqlfPreselectObject, sqlfInsertObject, sqlfSelectObject, sqlfDelete, sqlfUpdate, sqlfInsertInto, sqlfObjectProperty, sqlfSelectIds) + .filter(f -> null != f && !f.isEmpty()) + .forEach(script::append); + ret = new ParameterMapStatement(table.getSchema().getScope(), conn, script, remap); + } + else if (_dialect.isSqlServer()) + { + if (!parameters.isEmpty()) + { + SQLFragment select = new SQLFragment(); + sqlfDeclare.append("DECLARE "); + select.append("SELECT "); + comma = ""; + for (Map.Entry e : parameters.entrySet()) + { + ParameterHolder ph = e.getValue(); + sqlfDeclare.append(comma); + String variable = sqlServerVariableDeclaration(sqlfDeclare, ph); + select.append(comma).append(variable).append("=?"); + select.add(ph.p); + comma = ", "; + } + sqlfDeclare.appendEOS(); + sqlfDeclare.append(select); + sqlfDeclare.appendEOS(); + } + SQLFragment script = new SQLFragment(); + Stream.of(sqlfDeclare, sqlfPreselectObject, sqlfInsertObject, sqlfSelectObject, sqlfDelete, sqlfUpdate, sqlfInsertInto, sqlfObjectProperty, sqlfSelectIds) + .filter(f -> null != f && !f.isEmpty()) + .forEach(script::append); + _log.debug(script.toDebugString()); + ret = new ParameterMapStatement(table.getSchema().getScope(), conn, script, remap); + } + else + { + // wrap in a function with a single ROW() constructor argument + SQLFragment fn = new SQLFragment(); + String fnName = _dialect.getGlobalTempTablePrefix() + "fn_" + GUID.makeHash(); + TempTableTracker.track(fnName, fn); + + String typeName = fnName + "type"; + fn.append("CREATE TYPE ").append(typeName).append(" AS ("); + // TODO d.execute() doesn't handle temp schema + SQLFragment call = new SQLFragment(); + call.append(fnName).append("(ROW("); + comma = ""; + for (Map.Entry e : parameters.entrySet()) + { + ParameterHolder ph = e.getValue(); + String type = _dialect.getSqlTypeName(ph.p.getType()); + fn.append("\n").append(comma); + fn.append(makePgRowTypeName(ph.variableName)); + fn.append(" "); + fn.append(type); + // For PG (29687) we need the length for CHAR type + if (_dialect.isPostgreSQL() && JdbcType.CHAR.equals(ph.p.getType())) + fn.append("(").appendValue(ph.getScale()).append(")"); + call.append(comma).append("?"); + call.add(ph.p); + comma = ","; + } + fn.append("\n)").appendEOS(); + fn.append("CREATE FUNCTION ").append(fnName).append("(").append(typeName).append(") "); + fn.append("RETURNS "); + if (null != sqlfSelectIds) + fn.append("SETOF RECORD"); + else + fn.append("void"); + String quoteToken = "$x" + GUID.makeHash() + "$"; + fn.append(" AS ").append(quoteToken).append("\n"); + call.append("))"); + + if (null != sqlfSelectIds) + { + call.insert(0, "SELECT * FROM "); + call.append(" AS x("); + String sep = ""; + + if (_selectIds) + { + if (null != rowIdVar) + { + call.append("A BIGINT"); + sep = ", "; + } + if (null != objectIdVar) + { + call.append(sep); + call.append("B BIGINT"); + sep = ", "; + } + } + + if (_selectObjectUri && null != objectURIVar) + { + call.append(sep); + call.append("C VARCHAR"); + } + + call.append(")").appendEOS(); + } + else + { + call.insert(0, "{call "); + call.append("}"); + } + + fn.append(sqlfDeclare); + + fn.append("BEGIN\n"); + fn.append("-- ").append(_operation.name()).append("\n"); + Stream.of(sqlfPreselectObject, sqlfInsertObject, sqlfSelectObject, sqlfDelete, sqlfUpdate, sqlfInsertInto, sqlfObjectProperty) + .filter(f -> null != f && !f.isEmpty()) + .forEach(fn::append); + if (null == sqlfSelectIds) + { + fn.append("RETURN").appendEOS(); + } + else + { + sqlfSelectIds.insert(0, "RETURN QUERY\n"); + fn.append(sqlfSelectIds); + fn.appendEOS(); + } + fn.append("END").appendEOS().append(" ").append(quoteToken).append(" LANGUAGE plpgsql").appendEOS(); + _log.debug(fn.toDebugString()); + _log.debug(call.toDebugString()); + final SQLFragment drop = new SQLFragment("DROP TYPE IF EXISTS ").append(typeName).append(" CASCADE").appendEOS(); + _log.debug(drop.toDebugString()); + new SqlExecutor(table.getSchema()).execute(fn); + ret = new ParameterMapStatement(table.getSchema().getScope(), conn, call, updatable.remapSchemaColumns()); + ret.setDebugSql(fn.getSQL() + "--\n" + call.toDebugString()); + ret.onClose(() -> { + try + { + new SqlExecutor(ExperimentService.get().getSchema()).execute(drop); + } + catch (Exception x) + { + _log.error("Error dropping custom rowtype for temp function.", x); + } + }); + } + + int selectIndex = 1; + + if (_selectIds) + { + // Why is one of these boolean and the other an index?? I don't know + ret.setSelectRowId(selectAutoIncrement); + + if (selectAutoIncrement) + selectIndex++; + + if (null != objectIdVar) + ret.setObjectIdIndex(selectIndex++); + } + + if (_selectObjectUri && null != objectURIVar) + ret.setObjectUriIndex(selectIndex); + + return ret; + } + + private static LinkedHashMap getKeys( + UpdateableTableInfo updatable, + TableInfo table, + String objectURIColumnName, + Set keyColumnNames, + boolean preferPKOverObjectUriAsKey + ) + { + LinkedHashMap keys = new LinkedHashMap<>(); + ColumnInfo col = table.getColumn("Container"); + + if (null != col) + keys.put(col.getFieldKey(), col); + + if (null != keyColumnNames && !keyColumnNames.isEmpty()) + { + for (String name : keyColumnNames) + { + col = table.getColumn(name); + if (null == col) + throw new IllegalArgumentException("Column not found: " + name); + keys.put(col.getFieldKey(), col); + } + } + else + { + // using objectURIColumnName preferentially to be backward compatible with OntologyManager.saveTabDelimited + // which in turn is only called by LuminexDataHandler.saveDataRows() + col = objectURIColumnName == null ? null : table.getColumn(objectURIColumnName); + if (null != col && !preferPKOverObjectUriAsKey) + keys.put(col.getFieldKey(), col); + else + { + // See Issue 26661 and Issue 41053 + // NOTE: IMO we should not be using updatable.getPkColumnNames() here! If the caller doesn't want to use the + // 'real' PK from the SchemaTableInfo for update/merge, then the alternate keys should be explicitly specified + // using StatementUtils.keys() + for (String pkName : updatable.getPkColumnNames()) + { + col = table.getColumn(pkName); + if (null == col) + throw new IllegalStateException("pk column not found: " + pkName); + keys.put(col.getFieldKey(), col); + } + } + } + + return keys; + } + + private SQLFragment getPkWhereClause(LinkedHashMap keys) + { + SQLFragment sqlfWherePK = new SQLFragment(); + sqlfWherePK.append("\nWHERE "); + String and = ""; + for (Map.Entry e : keys.entrySet()) + { + ColumnInfo keyCol = e.getValue(); + ParameterHolder keyColPh = createParameter(keyCol); + + sqlfWherePK.append(and); + sqlfWherePK.append("("); + sqlfWherePK.appendIdentifier(keyCol.getSelectIdentifier()); + sqlfWherePK.append(" = "); + appendParameterOrVariable(sqlfWherePK, keyColPh); + if (keyCol.isNullable()) + { + sqlfWherePK.append(" OR "); + sqlfWherePK.appendIdentifier(keyCol.getSelectIdentifier()); + sqlfWherePK.append(" IS NULL AND "); + appendParameterOrVariable(sqlfWherePK, keyColPh); + sqlfWherePK.append(" IS NULL"); + } + sqlfWherePK.append(")"); + and = " AND "; + } + return sqlfWherePK; + } + + private String sqlServerVariableDeclaration(SQLFragment sqlfDeclare, ParameterHolder ph) + { + assert(_dialect.isSqlServer()); + String variable = ph.variableName; + sqlfDeclare.append(variable); + sqlfDeclare.append(" "); + JdbcType jdbcType = ph.p.getType(); + assert null != jdbcType; + String type = _dialect.getSqlTypeName(jdbcType); + assert null != type; + + // Workaround - SQLServer doesn't support TEXT, NTEXT, or IMAGE as local variables in statements, but is OK with NVARCHAR(MAX) + if (jdbcType.isText()) + { + if ("NTEXT".equalsIgnoreCase(type) || "TEXT".equalsIgnoreCase(type) || ph.getScale()>4000) + type = "NVARCHAR(MAX)"; + else + type = "NVARCHAR(4000)"; + } + // Add scale and precision for decimal values specifying scale + else if (jdbcType.isDecimal() && ph.getScale() > 0) + { + type = type + "(" + ph.getPrecision() + "," + ph.getScale() + ")"; + } + + sqlfDeclare.append(type); + return variable; + } + + /* + * We could use SQLFragment.appendValue() for most of these. However, here it is important to force + * the use of inline literal values. SQLFragment.appendValue() does not guarantee that. + */ + private void toLiteral(SQLFragment f, Object value) + { + if (null == value) + { + f.append("NULL"); + return; + } + if (value instanceof Number) + { + f.append(value.toString()); + return; + } + if (value instanceof SQLFragment.NowTimestamp now) + { + f.appendValue(now); + return; + } + if (value instanceof java.sql.Date sqlDate) + { + f.append("{d ").append(_dialect.getStringHandler().quoteStringLiteral(DateUtil.formatIsoDate(sqlDate))).append("}"); + return; + } + else if (value instanceof java.util.Date date) + { + f.append("{ts ").append(_dialect.getStringHandler().quoteStringLiteral(DateUtil.formatIsoDateShortTime(date))).append("}"); + return; + } + assert value instanceof String; + f.append(_dialect.getStringHandler().quoteStringLiteral(String.valueOf(value))); + } + + @SuppressWarnings("JUnitMalformedDeclaration") + public static class TestCase extends Assert + { + final static String DATA_CLASS_NAME = "StatementUtilsTestDataClass"; + final static String VOCAB_DOMAIN_KIND = "Vocabulary"; // VocabularyDomainKind.KIND_NAME + final static String VOCAB_DOMAIN_NAME = "StatementUtilsVocabularyDomain"; + + final Container container; + final TableInfo dataClassTable; + final TableInfo principalsTable; + final UpdateableTableInfo testTable; + final User user; + final Set vocabParameters = CaseInsensitiveHashSet.of("Age", "AgeMVIndicator", "Color", "ColorMVIndicator"); + final Set vocabProps; + + // Flag to run tests against one or both (Postgres, SQL Server) SqlDialects. Set to false by default + // since tests are run in both environments in CI. See "otherSqlDialect". + final boolean runOtherDialect = false; + + // This is a mock SqlDialect that mocks the alternative SqlDialect configuration to the current configuration. + // So, if the tests are running in a Postgres environment, then this represents a SQL Server SqlDialect + // and vice versa. This is useful for getting code coverage across code paths for both dialects in a single + // test run. Enabled via the "runOtherDialect" flag. + final SqlDialect otherSqlDialect; + + public TestCase() throws Exception + { + container = JunitUtil.getTestContainer(); + user = TestContext.get().getUser(); + + dataClassTable = QueryService.get().getUserSchema(user, container, ExpSchema.SCHEMA_EXP_DATA).getTableOrThrow(DATA_CLASS_NAME); + principalsTable = DbSchema.get("core", DbSchemaType.Module).getTable("principals"); + testTable = DbSchema.get("test", DbSchemaType.Module).getTable("testtable2"); + + // Initialize vocab domain properties + { + var vocabDomainKind = PropertyService.get().getDomainKindByName(VOCAB_DOMAIN_KIND); + var vocabDomainURI = vocabDomainKind.generateDomainURI(null, VOCAB_DOMAIN_NAME, container, user); + var vocabDomain = PropertyService.get().getDomain(container, vocabDomainURI); + assertNotNull(vocabDomain); + vocabProps = Set.of(vocabDomain.getPropertyByName("Age"), vocabDomain.getPropertyByName("Color")); + } + + if (runOtherDialect) + { + SqlDialect defaultDialect = principalsTable.getSqlDialect(); + boolean isPostgres = defaultDialect.isPostgreSQL(); + + otherSqlDialect = new MockSqlDialect() + { + @Override + public String addReselect(SQLFragment sql, ColumnInfo column, @Nullable String proposedVariable) + { + return defaultDialect.addReselect(sql, column, proposedVariable); + } + + @Override + public String getGuidType() + { + return defaultDialect.getGuidType(); + } + + @Override + public @Nullable String getSqlTypeName(JdbcType type) + { + return defaultDialect.getSqlTypeName(type); + } + + @Override + public boolean isPostgreSQL() + { + // Returns true in SQL Server configured environments + return !isPostgres; + } + + @Override + public boolean isSqlServer() + { + // Returns true in Postgres configured environments + return isPostgres; + } + }; + } + else + { + otherSqlDialect = null; + } + } + + @BeforeClass + public static void createDomains() throws Exception + { + var container = JunitUtil.getTestContainer(); + var user = TestContext.get().getUser(); + + // Create a data class domain + ExperimentService.get().createDataClass(container, user, DATA_CLASS_NAME, null, List.of(new GWTPropertyDescriptor("aa", "int")), List.of(), null, null); + + // Create a vocabulary domain + { + GWTPropertyDescriptor prop1 = new GWTPropertyDescriptor(); + prop1.setRangeURI("int"); + prop1.setName("Age"); + prop1.setMvEnabled(true); + + GWTPropertyDescriptor prop2 = new GWTPropertyDescriptor(); + prop2.setRangeURI("string"); + prop2.setName("Color"); + + GWTDomain domain = new GWTDomain<>(); + domain.setName(VOCAB_DOMAIN_NAME); + domain.setFields(List.of(prop1, prop2)); + + DomainUtil.createDomain(VOCAB_DOMAIN_KIND, domain, null, container, user, VOCAB_DOMAIN_NAME, null, false); + } + } + + @AfterClass + public static void cleanup() + { + deleteTestContainer(); + } + + @Test + public void testToLiteral() + { + boolean isPostgres = principalsTable.getSqlDialect().isPostgreSQL(); + + var statement = new StatementUtils(Operation.insert, principalsTable); + Function runToLiteral = (value) -> { + var sql = new SQLFragment(); + statement.toLiteral(sql, value); + return sql; + }; + + var dateLong = 1749759500016L; // Thu Jun 12 2025 13:18:20 GMT-0700 (Pacific Daylight Time) + + // null value + var actual = runToLiteral.apply(null); + assertEquals(new SQLFragment("NULL"), actual); + + // Number + actual = runToLiteral.apply(1234567890); + assertEquals(new SQLFragment("1234567890"), actual); + + // NowTimestamp + var now = new SQLFragment.NowTimestamp(dateLong); + actual = runToLiteral.apply(now); + assertEquals(new SQLFragment().appendValue(now), actual); + + // sql.Date + var sqlDate = new java.sql.Date(dateLong); + var dateFormat = new SimpleDateFormat(DateUtil.getStandardDateFormatString()); + var expected = String.format(isPostgres ? "{d '%s'}" : "{d N'%s'}", dateFormat.format(sqlDate)); + + actual = runToLiteral.apply(sqlDate); + assertEquals(new SQLFragment(expected), actual); + + // util.Date + var utilDate = new java.util.Date(dateLong); + dateFormat = new SimpleDateFormat(DateUtil.getStandardDateTimeFormatString()); + expected = String.format(isPostgres ? "{ts '%s'}" : "{ts N'%s'}", dateFormat.format(utilDate)); + + actual = runToLiteral.apply(utilDate); + assertEquals(new SQLFragment(expected), actual); + } + + @Test + public void testCreateStatementValidation() throws Exception + { + try (var conn = getConnection()) + { + var nonUpdateTable = new VirtualTable<>(DbSchema.get("test", DbSchemaType.Module), "virtualInsanity", null); + + var exception = Assert.assertThrows(IllegalArgumentException.class, () -> new StatementUtils(Operation.merge, nonUpdateTable).createStatement(conn, container, user)); + assertEquals("Table must be an UpdateableTableInfo", exception.getMessage()); + + // Unreachable with current mocks +// var nonDatabaseTable = QueryService.get().getUserSchema(user, container, "core").getTableOrThrow("Principals"); +// exception = Assert.assertThrows(IllegalArgumentException.class, () -> new StatementUtils(Operation.merge, nonDatabaseTable).createStatement(conn, container, user)); +// assertEquals("Table must be a database table", exception.getMessage()); + +// exception = Assert.assertThrows(IllegalArgumentException.class, () -> { +// var noIdentifierTable = principalsTable.getMetaDataIdentifier(). +// new StatementUtils(Operation.merge, nonDatabaseTable).dialect(new MockSqlDialect()).createStatement(conn, container, user); +// }); +// assertEquals("Table must have a metadata identifier", exception.getMessage()); + + exception = Assert.assertThrows(IllegalArgumentException.class, () -> new StatementUtils(Operation.merge, principalsTable).dialect(new MockSqlDialect()).createStatement(conn, container, user)); + assertEquals("Merge is only supported/tested on postgres and sql server", exception.getMessage()); + } + } + + @Test + public void testGetKeys() + { + var containerFieldKey = FieldKey.fromParts("Container"); + var rowIdFieldKey = FieldKey.fromParts("RowId"); + var textFieldKey = FieldKey.fromParts("Text"); + + var updateTable = testTable; + var table = updateTable.getSchemaTableInfo(); + + // Pre-conditions + var pkColumnNames = new CaseInsensitiveHashSet(testTable.getPkColumnNames()); + assertEquals(2, pkColumnNames.size()); + assertTrue(pkColumnNames.contains(containerFieldKey.getName())); + assertTrue(pkColumnNames.contains(textFieldKey.getName())); + assertNotNull(testTable.getColumn(rowIdFieldKey)); + + // The "Container" column is always resolved if present on the table + var keys = StatementUtils.getKeys(updateTable, table, null, Set.of(containerFieldKey.getName()), false); + assertEquals(1, keys.size()); + assertTrue(keys.containsKey(containerFieldKey)); + + // The "Container" column is only resolved even when in the explicit name map + keys = StatementUtils.getKeys(updateTable, table, null, Set.of(containerFieldKey.getName()), false); + assertEquals(1, keys.size()); + assertTrue(keys.containsKey(containerFieldKey)); + + // The "Container" column is also resolved even when not in the explicit key column map. Other key columns are included as well. + keys = StatementUtils.getKeys(updateTable, table, null, Set.of(textFieldKey.getName()), false); + assertEquals(2, keys.size()); + assertTrue(keys.containsKey(containerFieldKey)); + assertTrue(keys.containsKey(textFieldKey)); + + // All explicitly named columns should resolve as columns on the table + var exception = Assert.assertThrows(IllegalArgumentException.class, () -> StatementUtils.getKeys(updateTable, table, null, Set.of(textFieldKey.getName(), "Beep"), false)); + assertEquals("Column not found: Beep", exception.getMessage()); + + // Furnish an explicit "objectURIColumnName" and expect it to be included when preferPKOverObjectUriAsKey = false + keys = StatementUtils.getKeys(updateTable, table, "RowId", null, false); + assertEquals(2, keys.size()); + assertTrue(keys.containsKey(containerFieldKey)); + assertTrue(keys.containsKey(rowIdFieldKey)); + + // Furnish an explicit "objectURIColumnName" and expect it to NOT be included when preferPKOverObjectUriAsKey = true + keys = StatementUtils.getKeys(updateTable, table, "RowId", null, true); + assertEquals(2, keys.size()); + assertTrue(keys.containsKey(containerFieldKey)); + assertTrue(keys.containsKey(textFieldKey)); + + keys = StatementUtils.getKeys(updateTable, table, null, null, false); + assertEquals(2, keys.size()); + assertTrue(keys.containsKey(containerFieldKey)); + assertTrue(keys.containsKey(textFieldKey)); + } + + @Test + public void testInsert() throws Exception + { + ParameterMapStatement m = null; + try (Connection conn = getConnection()) + { + m = insertStatement(conn, principalsTable, container, user, true, true); + m.close(); m = null; + + m = insertStatement(conn, testTable, container, user, true, true); + m.close(); m = null; + } + finally + { + if (null != m) + m.close(); + } + } + + @Test + public void testInsertWithExtensibleDomain() throws Exception + { + ParameterMapStatement m = null; + try (Connection conn = getConnection(dataClassTable)) + { + StatementUtils statement; + + // Insert + { + var validateInsert = new Function() + { + @Override + public Object apply(StatementUtils s) + { + boolean isPostgres = s._dialect.isPostgreSQL(); + + assertTrue(s._columnTracker.insertColumns.contains("Created")); + assertTrue(s._columnTracker.insertColumns.contains("CreatedBy")); + assertTrue(s._columnTracker.insertColumns.contains("Modified")); + assertTrue(s._columnTracker.insertColumns.contains("ModifiedBy")); + assertTrue(s._columnTracker.updateColumns.isEmpty()); + + if (isPostgres) + { + assertTrue(s._columnTracker.selectColumns.contains("_rowid_")); + assertTrue(s._columnTracker.selectColumns.contains("_$objectid$_")); + } + else + { + // Variables are not used in SQL Server + assertFalse(s._columnTracker.selectColumns.contains("_rowid_")); + assertTrue(s._columnTracker.selectColumns.contains("@_objectid_")); + } + + var parameterKeys = s.parameters.keySet(); + assertTrue(vocabParameters.stream().noneMatch(parameterKeys::contains)); + return null; + } + }; + + statement = insertStatement(dataClassTable, true, true); + m = statement.createStatement(conn, container, user); + m.close(); + m = null; + validateInsert.apply(statement); + + if (runOtherDialect) + { + statement = insertStatement(dataClassTable, true, true); + statement.dialect(otherSqlDialect); + m = statement.createStatement(conn, container, user); + m.close(); + m = null; + validateInsert.apply(statement); + } + } + + // Insert with vocabulary properties + { + statement = insertStatement(dataClassTable, true, true); + statement.setVocabularyProperties(vocabProps); + m = statement.createStatement(conn, container, user); + m.close(); + m = null; + assertTrue(statement.parameters.keySet().containsAll(vocabParameters)); + } + } + finally + { + if (null != m) + m.close(); + } + } + + @Test + public void testUpdate() throws Exception + { + ParameterMapStatement m = null; + try (Connection conn = getConnection()) + { + m = updateStatement(conn, principalsTable, container, user, true, true); + m.close(); m = null; + + m = updateStatement(conn, testTable, container, user, true, true); + m.close(); m = null; + } + finally + { + if (null != m) + m.close(); + } + } + + @Test + public void testUpdateWithExtensibleDomain() throws Exception + { + ParameterMapStatement m = null; + try (Connection conn = getConnection(dataClassTable)) + { + StatementUtils statement; + + // Update + { + var validateUpdate = new Function() + { + @Override + public Object apply(StatementUtils s) + { + assertTrue(s._columnTracker.insertColumns.isEmpty()); + assertFalse(s._columnTracker.updateColumns.contains("Created")); + assertFalse(s._columnTracker.updateColumns.contains("CreatedBy")); + assertTrue(s._columnTracker.updateColumns.contains("Modified")); + assertTrue(s._columnTracker.updateColumns.contains("ModifiedBy")); + assertTrue(s._columnTracker.selectColumns.isEmpty()); + var parameterKeys = s.parameters.keySet(); + assertTrue(vocabParameters.stream().noneMatch(parameterKeys::contains)); + return null; + } + }; + + statement = updateStatement(dataClassTable, true, true); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + validateUpdate.apply(statement); + + Set allUpdateColumns = new CaseInsensitiveHashSet(statement._columnTracker.updateColumns); + + if (runOtherDialect) + { + statement = updateStatement(dataClassTable, true, true); + statement.dialect(otherSqlDialect); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + validateUpdate.apply(statement); + } + + statement = updateStatement(dataClassTable, false, false); + statement.noupdate(CaseInsensitiveHashSet.of("Modified", "ModifiedBy")); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + assertFalse(statement._columnTracker.updateColumns.contains("Modified")); + assertFalse(statement._columnTracker.updateColumns.contains("ModifiedBy")); + + statement = updateStatement(dataClassTable, false, false); + statement.noupdate(allUpdateColumns); + m = statement.createStatement(conn, container, user); + var debugSql = m.getDebugSql(); + m.close(); m = null; + assertTrue(debugSql.contains("'noop' WHERE 1 <> 1")); + } + + // Update with vocabulary properties + { + statement = updateStatement(dataClassTable, true, true); + statement.setVocabularyProperties(vocabProps); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + assertTrue(statement.parameters.keySet().containsAll(vocabParameters)); + } + } + finally + { + if (null != m) + m.close(); + } + } + + @Test + public void testUpdateWithObjectUriColumn() throws Exception + { + // Arrange + // Create a list + String listName = "StatementUtilsTestList"; + { + // Create a list domain + var listDef = ListService.get().createList(container, listName, ListDefinition.KeyType.AutoIncrementInteger); + listDef.setKeyName("pk"); + + Domain domain = requireNonNull(listDef.getDomain()); + addProperty(domain, "pk", PropertyType.INTEGER); + addProperty(domain, "name", PropertyType.STRING); + + listDef.save(user); + } + + ParameterMapStatement m = null; + TableInfo listTable = requireNonNull(ListService.get().getList(container, listName)).getTable(user, container); + assertNotNull(listTable); + + try (Connection conn = getConnection(listTable)) + { + StatementUtils statement; + var expectedNoUpdateColumns = CaseInsensitiveHashSet.of("EntityId", "Created", "CreatedBy"); + var expectedUpdateColumns = CaseInsensitiveHashSet.of("DIImportHash", "LastIndexed", "Modified", "ModifiedBy", "Name"); + + // Update statement (selectIds = true, autoFillDefaultColumns = true) + { + statement = StatementUtils.updateStatement(listTable, true, true); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + + // Assert + assertEquals(expectedNoUpdateColumns, statement._dontUpdateColumnNames); + assertTrue(statement._columnTracker.insertColumns.isEmpty()); + assertTrue(statement._columnTracker.selectColumns.isEmpty()); + + if (runOtherDialect) + { + statement = StatementUtils.updateStatement(listTable, true, true); + statement.dialect(otherSqlDialect); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + } + + assertEquals(expectedNoUpdateColumns, statement._dontUpdateColumnNames); + assertTrue(statement._columnTracker.insertColumns.isEmpty()); + assertTrue(statement._columnTracker.selectColumns.isEmpty()); + + assertEquals(expectedUpdateColumns, statement._columnTracker.updateColumns); + } + + // Update statement (selectIds = false, autoFillDefaultColumns = false) + { + statement = StatementUtils.updateStatement(listTable, false, false); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + + // Assert + assertEquals(expectedNoUpdateColumns, statement._dontUpdateColumnNames); + assertTrue(statement._columnTracker.insertColumns.isEmpty()); + assertEquals(expectedUpdateColumns, statement._columnTracker.updateColumns); + + if (runOtherDialect) + { + statement = StatementUtils.updateStatement(listTable, false, false); + statement.dialect(otherSqlDialect); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + } + + assertEquals(expectedNoUpdateColumns, statement._dontUpdateColumnNames); + assertTrue(statement._columnTracker.insertColumns.isEmpty()); + assertEquals(expectedUpdateColumns, statement._columnTracker.updateColumns); + } + } + finally + { + if (null != m) + m.close(); + } + } + + @Test + public void testMerge() throws Exception + { + ParameterMapStatement m = null; + try (Connection conn = getConnection()) + { + m = mergeStatement(conn, principalsTable, null, null, null, container, user, false, true, false); + m.close(); m = null; + + if (runOtherDialect) + { + StatementUtils statement = mergeStatement(principalsTable, null, null, null, false, true, false); + statement.dialect(otherSqlDialect); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + } + + m = mergeStatement(conn, testTable, null, null, null, container, user, false, true, false); + m.close(); m = null; + + if (runOtherDialect) + { + StatementUtils statement = mergeStatement(testTable, null, null, null, false, true, false); + statement.dialect(otherSqlDialect); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + } + } + finally + { + if (null != m) + m.close(); + } + } + + @Test + public void testMergeWithExtensibleDomain() throws Exception + { + ParameterMapStatement m = null; + try (Connection conn = getConnection(dataClassTable)) + { + StatementUtils statement; + + // Merge + { + var validateMerge = new Function() + { + @Override + public Object apply(StatementUtils s) + { + boolean isPostgres = s._dialect.isPostgreSQL(); + + assertTrue(s._columnTracker.insertColumns.contains("Container")); + assertTrue(s._columnTracker.insertColumns.contains("LSID")); + assertFalse(s._columnTracker.updateColumns.contains("Container")); + assertFalse(s._columnTracker.updateColumns.contains("LSID")); + + if (isPostgres) + { + assertTrue(s._columnTracker.selectColumns.contains("_rowid_")); + assertTrue(s._columnTracker.selectColumns.contains("_$objectid$_")); + } + else + { + // Variables are not used in SQL Server + assertFalse(s._columnTracker.selectColumns.contains("_rowid_")); + assertTrue(s._columnTracker.selectColumns.contains("@_objectid_")); + } + + var parameterKeys = s.parameters.keySet(); + assertTrue(vocabParameters.stream().noneMatch(parameterKeys::contains)); + return null; + } + }; + + statement = mergeStatement(dataClassTable, null, null, null, true, true, false); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + validateMerge.apply(statement); + + var updateColumns = new CaseInsensitiveHashSet(statement._columnTracker.updateColumns); + + if (runOtherDialect) + { + statement = mergeStatement(dataClassTable, null, null, null, true, true, false); + statement.dialect(otherSqlDialect); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + validateMerge.apply(statement); + } + + // TODO: This generates a SQL parsing error in Postgres due to the reselect statement coming before the WHERE clause +// statement = mergeStatement(dataClassTable, null, CaseInsensitiveHashSet.of("RunId"), updateColumns, true, true, false); +// m = statement.createStatement(conn, container, user); +// m.close(); m = null; +// assertTrue(statement._columnTracker.updateColumns.isEmpty()); + } + + // Merge with vocabulary properties + { + statement = mergeStatement(dataClassTable, null, null, null, false, true, false); + statement.setVocabularyProperties(vocabProps); + m = statement.createStatement(conn, container, user); + m.close(); m = null; + assertTrue(statement.parameters.keySet().containsAll(vocabParameters)); + } + } + finally + { + if (null != m) + m.close(); + } + } + + private static void addProperty(Domain d, String name, PropertyType pt) + { + DomainProperty p = d.addProperty(); + p.setName(name); + p.setPropertyURI(d.getTypeURI() + "#" + name); + p.setRangeURI(pt.getTypeUri()); + } + + private Connection getConnection() throws SQLException + { + return getConnection(principalsTable); + } + + private Connection getConnection(TableInfo table) throws SQLException + { + return table.getSchema().getScope().getConnection(); + } + } +} diff --git a/audit/src/org/labkey/audit/AuditLogImpl.java b/audit/src/org/labkey/audit/AuditLogImpl.java index 60d33f08858..dd38dba1fbe 100644 --- a/audit/src/org/labkey/audit/AuditLogImpl.java +++ b/audit/src/org/labkey/audit/AuditLogImpl.java @@ -1,316 +1,316 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.audit; - -import jakarta.servlet.ServletContext; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.AuditTypeProvider; -import org.labkey.api.audit.DetailedAuditTypeEvent; -import org.labkey.api.audit.SampleTimelineAuditEvent; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Sort; -import org.labkey.api.data.TableSelector; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.UserSchema; -import org.labkey.api.security.User; -import org.labkey.api.security.UserManager; -import org.labkey.api.util.ContextListener; -import org.labkey.api.util.Pair; -import org.labkey.api.util.StartupListener; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.HttpView; -import org.labkey.audit.model.LogManager; -import org.labkey.audit.query.AuditQuerySchema; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - -public class AuditLogImpl implements AuditLogService, StartupListener -{ - private static final AuditLogImpl _instance = new AuditLogImpl(); - - private static final Logger _log = LogHelper.getLogger(AuditLogImpl.class, "Audit service interactions."); - - private final Queue> _eventTypeQueue = new LinkedList<>(); - private final AtomicBoolean _logToDatabase = new AtomicBoolean(false); - private static final Object STARTUP_LOCK = new Object(); - - // Cache the audit events associated with transaction ids. We currently use these for interacting with objects - // that were created immediately after they were created, so the cache size does not need to be very large and the defaultTimeToLive can be small. - // Use a pair as the cache object to avoid warnings about mutable cache objects (Issue 48779). - // Since this is all about capturing data from the same transaction, there shouldn't be other threads in the mix. - private static final Cache>> TRANSACTION_EVENT_CACHE = CacheManager.getBlockingCache(50, CacheManager.HOUR, - "Transaction Audit Event Cache", - (key, argument) -> Pair.of(key, new ArrayList<>()) - ); - - public static AuditLogImpl get() - { - return _instance; - } - - private AuditLogImpl() - { - // If we're migrating, avoid creating all the audit log tables and inserting the queued events - if (ModuleLoader.getInstance().shouldInsertData()) - ContextListener.addStartupListener(this); - } - - @Override - public String getName() - { - return "Audit Log"; - } - - @Override - public void moduleStartupComplete(ServletContext servletContext) - { - // perform audit provider initialization - for (AuditTypeProvider provider : AuditLogService.get().getAuditProviders()) - { - provider.initializeProvider(User.getAdminServiceUser()); - } - - // Synchronize so that we can guarantee that all events have already been added to the queue before we - // start processing them - synchronized (STARTUP_LOCK) - { - _logToDatabase.set(true); - } - - while (!_eventTypeQueue.isEmpty()) - { - Pair event = _eventTypeQueue.remove(); - addEvents(event.first, List.of(event.second)); - } - } - - @Override - public boolean isViewable() - { - return true; - } - - @Override - public K addEvent(User user, K event) - { - return _addEvents(user, List.of(event),true, false); - } - - @Override - public void addEvents(@Nullable User user, List events) - { - _addEvents(user, events, false, false); - } - - @Override - public void addEvents(@Nullable User user, List events, boolean useTransactionAuditCache) - { - _addEvents(user, events, false, useTransactionAuditCache); - } - - private K _addEvents(@Nullable User user, List events, boolean reselectEvent, boolean useTransactionAuditCache) - { - assert !reselectEvent || events.size() == 1; - - for (var event : events) - { - assert event.getContainer() != null : "Container cannot be null"; - - if (user == null) - { - if (HttpView.hasCurrentView() && HttpView.currentContext() != null) - _log.warn("user was not specified for event type " + event.getEventType() + " in container " + event.getContainer() + "; defaulting to guest user."); - user = UserManager.getGuestUser(); - } - if (event.getTransactionId() != null && useTransactionAuditCache) - { - List transactionEvents = TRANSACTION_EVENT_CACHE.get(event.getTransactionId()).second; - transactionEvents.add(event); - } - - if (event.getImpersonatedBy() == null && user.isImpersonated()) - { - User impersonatingUser = user.getImpersonatingUser(); - event.setImpersonatedBy(impersonatingUser.getUserId()); - } - } - - try (var ignored = SpringActionController.ignoreSqlUpdates()) - { - /* - This is necessary because audit log service needs to be registered in the constructor - of the audit module, but the schema may not be created or updated at that point. Events - that occur before startup is complete are therefore queued up and recorded after startup. - */ - boolean databaseReady; - synchronized (STARTUP_LOCK) - { - // Keep the critical section as lean as possible - just guarantee that all the events - // have been queued before releasing the lock - databaseReady = _logToDatabase.get(); - if (!databaseReady) - { - for (var event : events) - _eventTypeQueue.add(new Pair<>(user, event)); - } - } - - if (databaseReady) - { - if (reselectEvent && events.size()==1) - return LogManager.get().insertEvent(user, events.get(0)); - LogManager.get().insertEvents(user, events); - } - } - catch (RuntimeException e) - { - _log.error("Failed to insert audit log event", e); - AuditLogService.handleAuditFailure(user, e); - throw e; - } - return null; - } - - @Override - public UserSchema createSchema(User user, Container container) - { - return new AuditQuerySchema(user, container); - } - - @Nullable - @Override - public K getAuditEvent(User user, String eventType, int rowId) - { - return LogManager.get().getAuditEvent(user, eventType, rowId); - } - - @Nullable - @Override - public K getAuditEvent(User user, String eventType, int rowId, @Nullable ContainerFilter cf) - { - return LogManager.get().getAuditEvent(user, eventType, rowId, cf); - } - - @Override - public List getAuditEvents(Container container, User user, String eventType, @Nullable SimpleFilter filter, @Nullable Sort sort) - { - return LogManager.get().getAuditEvents(container, user, eventType, filter, sort); - } - - @Override - public List getAuditEvents(Container container, User user, String eventType, @Nullable SimpleFilter filter, @Nullable Sort sort, @Nullable ContainerFilter cf) - { - return LogManager.get().getAuditEvents(container, user, eventType, filter, sort, cf); - } - - @Override - public ActionURL getAuditUrl() - { - return new ActionURL(AuditController.ShowAuditLogAction.class, ContainerManager.getRoot()); - } - - public List getTransactionSampleIds(long transactionAuditId, User user, Container container, @Nullable ContainerFilter containerFilter) - { - List transactionEvents = TRANSACTION_EVENT_CACHE.get(transactionAuditId).second; - if (!transactionEvents.isEmpty()) - { - List ids = new ArrayList<>(); - transactionEvents.forEach(event -> { - if (event instanceof SampleTimelineAuditEvent stEvent) - ids.add(stEvent.getSampleId()); - }); - return ids; - } - - SimpleFilter filter = new SimpleFilter(); - filter.addCondition(FieldKey.fromParts("TransactionID"), transactionAuditId); - - List events = AuditLogService.get().getAuditEvents(container, user, SampleTimelineAuditEvent.EVENT_TYPE, filter, null, containerFilter); - return events.stream().map(SampleTimelineAuditEvent::getSampleId).collect(Collectors.toList()); - } - - public List getTransactionSourceIds(long transactionAuditId, User user, Container container, @Nullable ContainerFilter containerFilter) - { - List lsids = new ArrayList<>(); - List sourceIds = new ArrayList<>(); - List transactionEvents = TRANSACTION_EVENT_CACHE.get(transactionAuditId).second; - if (!transactionEvents.isEmpty()) - { - transactionEvents.forEach(event -> { - if (event instanceof DetailedAuditTypeEvent detailedEvent) - { - if (detailedEvent.getNewRecordMap() != null) - { - Map newRecord = new CaseInsensitiveHashMap<>(AbstractAuditTypeProvider.decodeFromDataMap(detailedEvent.getNewRecordMap())); - if (newRecord.containsKey("RowId") && !StringUtils.isEmpty(newRecord.get("RowId"))) - sourceIds.add(Long.valueOf(newRecord.get("RowId"))); - else if (newRecord.containsKey("LSID") && !StringUtils.isEmpty(newRecord.get("LSID"))) - lsids.add(newRecord.get("LSID")); - } - } - }); - } - else - { - List events = QueryService.get().getQueryUpdateAuditRecords(user, container, transactionAuditId, containerFilter); - - events.forEach((event) -> { - if (event.getNewRecordMap() != null) - { - Map newRecord = new CaseInsensitiveHashMap<>(AbstractAuditTypeProvider.decodeFromDataMap(event.getNewRecordMap())); - if (newRecord.containsKey("RowId") && !StringUtils.isEmpty(newRecord.get("RowId"))) - sourceIds.add(Long.valueOf(newRecord.get("RowId"))); - else if (newRecord.containsKey("LSID") && !StringUtils.isEmpty(newRecord.get("LSID"))) - lsids.add(newRecord.get("LSID")); - - } - }); - } - if (!lsids.isEmpty()) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromParts("LSID"), lsids, CompareType.IN); - TableSelector selector = new TableSelector(ExperimentService.get().getTinfoData(), Collections.singleton("RowId"), filter, null); - sourceIds.addAll(selector.getArrayList(Long.class)); - } - return sourceIds; - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.audit; + +import jakarta.servlet.ServletContext; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.AuditTypeProvider; +import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.SampleTimelineAuditEvent; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; +import org.labkey.api.data.TableSelector; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.util.ContextListener; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StartupListener; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.HttpView; +import org.labkey.audit.model.LogManager; +import org.labkey.audit.query.AuditQuerySchema; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +public class AuditLogImpl implements AuditLogService, StartupListener +{ + private static final AuditLogImpl _instance = new AuditLogImpl(); + + private static final Logger _log = LogHelper.getLogger(AuditLogImpl.class, "Audit service interactions."); + + private final Queue> _eventTypeQueue = new LinkedList<>(); + private final AtomicBoolean _logToDatabase = new AtomicBoolean(false); + private static final Object STARTUP_LOCK = new Object(); + + // Cache the audit events associated with transaction ids. We currently use these for interacting with objects + // that were created immediately after they were created, so the cache size does not need to be very large and the defaultTimeToLive can be small. + // Use a pair as the cache object to avoid warnings about mutable cache objects (Issue 48779). + // Since this is all about capturing data from the same transaction, there shouldn't be other threads in the mix. + private static final Cache>> TRANSACTION_EVENT_CACHE = CacheManager.getBlockingCache(50, CacheManager.HOUR, + "Transaction Audit Event Cache", + (key, argument) -> Pair.of(key, new ArrayList<>()) + ); + + public static AuditLogImpl get() + { + return _instance; + } + + private AuditLogImpl() + { + // If we're migrating, avoid creating all the audit log tables and inserting the queued events + if (ModuleLoader.getInstance().shouldInsertData()) + ContextListener.addStartupListener(this); + } + + @Override + public String getName() + { + return "Audit Log"; + } + + @Override + public void moduleStartupComplete(ServletContext servletContext) + { + // perform audit provider initialization + for (AuditTypeProvider provider : AuditLogService.get().getAuditProviders()) + { + provider.initializeProvider(User.getAdminServiceUser()); + } + + // Synchronize so that we can guarantee that all events have already been added to the queue before we + // start processing them + synchronized (STARTUP_LOCK) + { + _logToDatabase.set(true); + } + + while (!_eventTypeQueue.isEmpty()) + { + Pair event = _eventTypeQueue.remove(); + addEvents(event.first, List.of(event.second)); + } + } + + @Override + public boolean isViewable() + { + return true; + } + + @Override + public K addEvent(User user, K event) + { + return _addEvents(user, List.of(event),true, false); + } + + @Override + public void addEvents(@Nullable User user, List events) + { + _addEvents(user, events, false, false); + } + + @Override + public void addEvents(@Nullable User user, List events, boolean useTransactionAuditCache) + { + _addEvents(user, events, false, useTransactionAuditCache); + } + + private K _addEvents(@Nullable User user, List events, boolean reselectEvent, boolean useTransactionAuditCache) + { + assert !reselectEvent || events.size() == 1; + + for (var event : events) + { + assert event.getContainer() != null : "Container cannot be null"; + + if (user == null) + { + if (HttpView.hasCurrentView() && HttpView.currentContext() != null) + _log.warn("user was not specified for event type " + event.getEventType() + " in container " + event.getContainer() + "; defaulting to guest user."); + user = UserManager.getGuestUser(); + } + if (event.getTransactionId() != null && useTransactionAuditCache) + { + List transactionEvents = TRANSACTION_EVENT_CACHE.get(event.getTransactionId()).second; + transactionEvents.add(event); + } + + if (event.getImpersonatedBy() == null && user.isImpersonated()) + { + User impersonatingUser = user.getImpersonatingUser(); + event.setImpersonatedBy(impersonatingUser.getUserId()); + } + } + + try (var ignored = SpringActionController.ignoreSqlUpdates()) + { + /* + This is necessary because audit log service needs to be registered in the constructor + of the audit module, but the schema may not be created or updated at that point. Events + that occur before startup is complete are therefore queued up and recorded after startup. + */ + boolean databaseReady; + synchronized (STARTUP_LOCK) + { + // Keep the critical section as lean as possible - just guarantee that all the events + // have been queued before releasing the lock + databaseReady = _logToDatabase.get(); + if (!databaseReady) + { + for (var event : events) + _eventTypeQueue.add(new Pair<>(user, event)); + } + } + + if (databaseReady) + { + if (reselectEvent && events.size()==1) + return LogManager.get().insertEvent(user, events.get(0)); + LogManager.get().insertEvents(user, events); + } + } + catch (RuntimeException e) + { + _log.error("Failed to insert audit log event", e); + AuditLogService.handleAuditFailure(user, e); + throw e; + } + return null; + } + + @Override + public UserSchema createSchema(User user, Container container) + { + return new AuditQuerySchema(user, container); + } + + @Nullable + @Override + public K getAuditEvent(User user, String eventType, int rowId) + { + return LogManager.get().getAuditEvent(user, eventType, rowId); + } + + @Nullable + @Override + public K getAuditEvent(User user, String eventType, int rowId, @Nullable ContainerFilter cf) + { + return LogManager.get().getAuditEvent(user, eventType, rowId, cf); + } + + @Override + public List getAuditEvents(Container container, User user, String eventType, @Nullable SimpleFilter filter, @Nullable Sort sort) + { + return LogManager.get().getAuditEvents(container, user, eventType, filter, sort); + } + + @Override + public List getAuditEvents(Container container, User user, String eventType, @Nullable SimpleFilter filter, @Nullable Sort sort, @Nullable ContainerFilter cf) + { + return LogManager.get().getAuditEvents(container, user, eventType, filter, sort, cf); + } + + @Override + public ActionURL getAuditUrl() + { + return new ActionURL(AuditController.ShowAuditLogAction.class, ContainerManager.getRoot()); + } + + public List getTransactionSampleIds(long transactionAuditId, User user, Container container, @Nullable ContainerFilter containerFilter) + { + List transactionEvents = TRANSACTION_EVENT_CACHE.get(transactionAuditId).second; + if (!transactionEvents.isEmpty()) + { + List ids = new ArrayList<>(); + transactionEvents.forEach(event -> { + if (event instanceof SampleTimelineAuditEvent stEvent) + ids.add(stEvent.getSampleId()); + }); + return ids; + } + + SimpleFilter filter = new SimpleFilter(); + filter.addCondition(FieldKey.fromParts("TransactionID"), transactionAuditId); + + List events = AuditLogService.get().getAuditEvents(container, user, SampleTimelineAuditEvent.EVENT_TYPE, filter, null, containerFilter); + return events.stream().map(SampleTimelineAuditEvent::getSampleId).collect(Collectors.toList()); + } + + public List getTransactionSourceIds(long transactionAuditId, User user, Container container, @Nullable ContainerFilter containerFilter) + { + List lsids = new ArrayList<>(); + List sourceIds = new ArrayList<>(); + List transactionEvents = TRANSACTION_EVENT_CACHE.get(transactionAuditId).second; + if (!transactionEvents.isEmpty()) + { + transactionEvents.forEach(event -> { + if (event instanceof DetailedAuditTypeEvent detailedEvent) + { + if (detailedEvent.getNewRecordMap() != null) + { + Map newRecord = new CaseInsensitiveHashMap<>(AbstractAuditTypeProvider.decodeFromDataMap(detailedEvent.getNewRecordMap())); + if (newRecord.containsKey("RowId") && !StringUtils.isEmpty(newRecord.get("RowId"))) + sourceIds.add(Long.valueOf(newRecord.get("RowId"))); + else if (newRecord.containsKey("LSID") && !StringUtils.isEmpty(newRecord.get("LSID"))) + lsids.add(newRecord.get("LSID")); + } + } + }); + } + else + { + List events = QueryService.get().getQueryUpdateAuditRecords(user, container, transactionAuditId, containerFilter); + + events.forEach((event) -> { + if (event.getNewRecordMap() != null) + { + Map newRecord = new CaseInsensitiveHashMap<>(AbstractAuditTypeProvider.decodeFromDataMap(event.getNewRecordMap())); + if (newRecord.containsKey("RowId") && !StringUtils.isEmpty(newRecord.get("RowId"))) + sourceIds.add(Long.valueOf(newRecord.get("RowId"))); + else if (newRecord.containsKey("LSID") && !StringUtils.isEmpty(newRecord.get("LSID"))) + lsids.add(newRecord.get("LSID")); + + } + }); + } + if (!lsids.isEmpty()) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromParts("LSID"), lsids, CompareType.IN); + TableSelector selector = new TableSelector(ExperimentService.get().getTinfoData(), Collections.singleton("RowId"), filter, null); + sourceIds.addAll(selector.getArrayList(Long.class)); + } + return sourceIds; + } +} diff --git a/audit/src/org/labkey/audit/model/LogManager.java b/audit/src/org/labkey/audit/model/LogManager.java index 1ee60f5b965..801701f1db4 100644 --- a/audit/src/org/labkey/audit/model/LogManager.java +++ b/audit/src/org/labkey/audit/model/LogManager.java @@ -1,289 +1,289 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.audit.model; - -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.AuditTypeProvider; -import org.labkey.api.audit.query.DefaultAuditTypeTable; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.ObjectFactory; -import org.labkey.api.data.ParameterMapStatement; -import org.labkey.api.data.PropertyStorageSpec; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Sort; -import org.labkey.api.data.StatementUtils; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainKind; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.query.UserSchema; -import org.labkey.api.security.User; -import org.labkey.api.view.HttpView; -import org.labkey.audit.AuditSchema; - -import java.sql.Connection; -import java.sql.SQLException; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; - -public class LogManager -{ - private static final Logger _log = org.apache.logging.log4j.LogManager.getLogger(LogManager.class); - private static final LogManager _instance = new LogManager(); - - private LogManager(){} - static public LogManager get() - { - return _instance; - } - - public DbSchema getSchema() - { - return AuditSchema.getInstance().getSchema(); - } - - /** There are a few places that depend on the reselect behavior. e.g. to get the rowid of the event */ - public K insertEvent(User user, K type) - { - Logger auditLogger = getAuditLogger(type); - auditLogger.info(type.getAuditLogMessage()); - - AuditTypeProvider provider = AuditLogService.get().getAuditProvider(type.getEventType()); - - if (provider != null) - { - Container c = type.getContainer(); - - UserSchema schema = AuditLogService.getAuditLogSchema(user, c != null ? c : ContainerManager.getRoot()); - - if (schema != null) - { - TableInfo table = schema.getTable(provider.getEventName(), false); - - if (table instanceof DefaultAuditTypeTable auditTypeTable) - { - // consider using etl data iterator for inserts - type = validateFields(provider, type); - TableInfo dbTable = auditTypeTable.getRealTable(); - K ret = Table.insert(user, dbTable, type); - return ret; - } - } - } - return null; - } - - private static Logger getAuditLogger(K type) - { - return org.apache.logging.log4j.LogManager.getLogger("org.labkey.audit.event." + type.getEventType().replaceAll(" ", "")); - } - - /** all events must be of same event type and container, for optimized code path */ - public void insertEvents(User user, List events) - { - if (events.isEmpty()) - return; - - AuditTypeEvent type = events.get(0); - - // Out of an abundance of caution and backward compatible behavior, do one-at-a-time logging if - // there is no transaction. Can revisit if this is not necessary. - // Keep in mind that the audit schema might not be in the same scope as table that is being logged about. - boolean optimize = events.size() == 1 || getSchema().getScope().isTransactionActive(); - if (optimize) - { - // make sure all events are the same type - final String expectedEventType = type.getEventType(); - final Container expectedContainer = type.getContainer(); - Optional problemEvent = events.stream() - .filter(event -> !Objects.equals(expectedEventType, event.getEventType()) || !Objects.equals(expectedContainer, event.getContainer())) - .findAny(); - optimize = problemEvent.isEmpty(); - } - - if (!optimize) - { - // do one at a time if events are not all the same - for (var event : events) - insertEvent(user, event); - return; - } - - AuditTypeProvider provider = AuditLogService.get().getAuditProvider(type.getEventType()); - if (null == provider) - return; - Container c = type.getContainer(); - UserSchema schema = AuditLogService.getAuditLogSchema(user, c != null ? c : ContainerManager.getRoot()); - TableInfo table = null==schema ? null : schema.getTable(provider.getEventName(), false); - TableInfo dbTable = table instanceof DefaultAuditTypeTable auditTypeTable ? auditTypeTable.getRealTable() : null; - - Logger auditLogger = getAuditLogger(type); - SQLException sqlx = null; - - if (null != dbTable) - { - try (Connection conn = dbTable.getSchema().getScope().getConnection()) - { - ParameterMapStatement stmt = StatementUtils.insertStatement(conn, dbTable, c, user, false, true); - for (var event : events) - { - event = validateFields(provider, event); - Map map = ObjectFactory.Registry.getFactory((Class)event.getClass()).toMap(event, null); - stmt.clearParameters(); - stmt.putAll(map); - stmt.addBatch(); - } - stmt.executeBatch(); - } - catch (SQLException x) - { - auditLogger.warn("Error occurred saving audit entries to database"); - sqlx = x; - } - } - - if (auditLogger.isInfoEnabled()) - { - // CONSIDER log these in TX.addCommitTask()? (but then what if updates are happening in a different scope?) - for (var event : events) - auditLogger.info(event.getAuditLogMessage()); - } - - if (null != sqlx) - throw new RuntimeSQLException(sqlx); - } - - @Nullable - public K getAuditEvent(User user, String eventType, int rowId, @Nullable ContainerFilter cf) - { - AuditTypeProvider provider = AuditLogService.get().getAuditProvider(eventType); - if (provider != null) - { - UserSchema schema = AuditLogService.getAuditLogSchema(user, HttpView.currentContext().getContainer()); - - if (schema != null) - { - TableInfo table = schema.getTable(provider.getEventName(), cf); - TableSelector selector = new TableSelector(table, null, null); - - return selector.getObject(rowId, provider.getEventClass()); - } - } - return null; - } - - @Nullable - public K getAuditEvent(User user, String eventType, int rowId) - { - return getAuditEvent(user, eventType, rowId, null); - } - - public List getAuditEvents(Container container, User user, String eventType, @Nullable SimpleFilter filter, @Nullable Sort sort) - { - return getAuditEvents(container, user, eventType, filter, sort, null); - } - public List getAuditEvents(Container container, User user, String eventType, @Nullable SimpleFilter filter, @Nullable Sort sort, @Nullable ContainerFilter cf) - { - AuditTypeProvider provider = AuditLogService.get().getAuditProvider(eventType); - if (provider != null) - { - UserSchema schema = AuditLogService.getAuditLogSchema(user, container); - - if (schema != null) - { - TableInfo table = schema.getTable(provider.getEventName(), cf); - TableSelector selector = new TableSelector(table, filter, sort); - - return selector.getArrayList(provider.getEventClass()); - } - } - return Collections.emptyList(); - } - - /** - * Ensure that the string properties don't exceed the length of the provisioned columns. - * Values will be trimmed to the max length. - */ - private K validateFields(@NotNull AuditTypeProvider provider, @NotNull K type) - { - ObjectFactory factory = ObjectFactory.Registry.getFactory((Class)type.getClass()); - Map values = new CaseInsensitiveHashMap<>(); - factory.toMap(type, values); - - boolean changed = false; - Domain domain = provider.getDomain(); - - DomainKind domainKind = domain.getDomainKind(); - for (PropertyStorageSpec prop : domainKind.getBaseProperties(domain)) - { - Object value = values.get(prop.getName()); - if (prop.getJdbcType().isText() && value instanceof String s) - { - int scale = prop.getSize(); - if (s.length() > scale) - { - _log.warn("Audit field input : \n" + prop.getName() + "\nexceeded the maximum length : " + scale); - String trimmed = s.substring(0, scale-3) + "..."; - values.put(prop.getName(), trimmed); - changed = true; - } - } - } - - for (DomainProperty dp : domain.getProperties()) - { - // For now, only check for string length like we were doing for the old audit event fields - PropertyDescriptor pd = dp.getPropertyDescriptor(); - Object value = values.get(dp.getName()); - if (pd.isStringType() && value instanceof String s) - { - int scale = dp.getScale(); - if (scale > 0 && s.length() > scale) - { - _log.warn("Audit field input : \n" + pd.getName() + "\nexceeded the maximum length : " + scale); - String trimmed; - if (scale > 100) - trimmed = s.substring(0, scale-3) + "..."; - else - trimmed = s.substring(0, scale); - values.put(pd.getName(), trimmed); - changed = true; - } - } - } - - if (changed) - return factory.fromMap(values); - else - return type; - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.audit.model; + +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.AuditTypeProvider; +import org.labkey.api.audit.query.DefaultAuditTypeTable; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.ObjectFactory; +import org.labkey.api.data.ParameterMapStatement; +import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; +import org.labkey.api.data.StatementUtils; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainKind; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; +import org.labkey.api.view.HttpView; +import org.labkey.audit.AuditSchema; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +public class LogManager +{ + private static final Logger _log = org.apache.logging.log4j.LogManager.getLogger(LogManager.class); + private static final LogManager _instance = new LogManager(); + + private LogManager(){} + static public LogManager get() + { + return _instance; + } + + public DbSchema getSchema() + { + return AuditSchema.getInstance().getSchema(); + } + + /** There are a few places that depend on the reselect behavior. e.g. to get the rowid of the event */ + public K insertEvent(User user, K type) + { + Logger auditLogger = getAuditLogger(type); + auditLogger.info(type.getAuditLogMessage()); + + AuditTypeProvider provider = AuditLogService.get().getAuditProvider(type.getEventType()); + + if (provider != null) + { + Container c = type.getContainer(); + + UserSchema schema = AuditLogService.getAuditLogSchema(user, c != null ? c : ContainerManager.getRoot()); + + if (schema != null) + { + TableInfo table = schema.getTable(provider.getEventName(), false); + + if (table instanceof DefaultAuditTypeTable auditTypeTable) + { + // consider using etl data iterator for inserts + type = validateFields(provider, type); + TableInfo dbTable = auditTypeTable.getRealTable(); + K ret = Table.insert(user, dbTable, type); + return ret; + } + } + } + return null; + } + + private static Logger getAuditLogger(K type) + { + return org.apache.logging.log4j.LogManager.getLogger("org.labkey.audit.event." + type.getEventType().replaceAll(" ", "")); + } + + /** all events must be of same event type and container, for optimized code path */ + public void insertEvents(User user, List events) + { + if (events.isEmpty()) + return; + + AuditTypeEvent type = events.get(0); + + // Out of an abundance of caution and backward compatible behavior, do one-at-a-time logging if + // there is no transaction. Can revisit if this is not necessary. + // Keep in mind that the audit schema might not be in the same scope as table that is being logged about. + boolean optimize = events.size() == 1 || getSchema().getScope().isTransactionActive(); + if (optimize) + { + // make sure all events are the same type + final String expectedEventType = type.getEventType(); + final Container expectedContainer = type.getContainer(); + Optional problemEvent = events.stream() + .filter(event -> !Objects.equals(expectedEventType, event.getEventType()) || !Objects.equals(expectedContainer, event.getContainer())) + .findAny(); + optimize = problemEvent.isEmpty(); + } + + if (!optimize) + { + // do one at a time if events are not all the same + for (var event : events) + insertEvent(user, event); + return; + } + + AuditTypeProvider provider = AuditLogService.get().getAuditProvider(type.getEventType()); + if (null == provider) + return; + Container c = type.getContainer(); + UserSchema schema = AuditLogService.getAuditLogSchema(user, c != null ? c : ContainerManager.getRoot()); + TableInfo table = null==schema ? null : schema.getTable(provider.getEventName(), false); + TableInfo dbTable = table instanceof DefaultAuditTypeTable auditTypeTable ? auditTypeTable.getRealTable() : null; + + Logger auditLogger = getAuditLogger(type); + SQLException sqlx = null; + + if (null != dbTable) + { + try (Connection conn = dbTable.getSchema().getScope().getConnection()) + { + ParameterMapStatement stmt = StatementUtils.insertStatement(conn, dbTable, c, user, false, true); + for (var event : events) + { + event = validateFields(provider, event); + Map map = ObjectFactory.Registry.getFactory((Class)event.getClass()).toMap(event, null); + stmt.clearParameters(); + stmt.putAll(map); + stmt.addBatch(); + } + stmt.executeBatch(); + } + catch (SQLException x) + { + auditLogger.warn("Error occurred saving audit entries to database"); + sqlx = x; + } + } + + if (auditLogger.isInfoEnabled()) + { + // CONSIDER log these in TX.addCommitTask()? (but then what if updates are happening in a different scope?) + for (var event : events) + auditLogger.info(event.getAuditLogMessage()); + } + + if (null != sqlx) + throw new RuntimeSQLException(sqlx); + } + + @Nullable + public K getAuditEvent(User user, String eventType, int rowId, @Nullable ContainerFilter cf) + { + AuditTypeProvider provider = AuditLogService.get().getAuditProvider(eventType); + if (provider != null) + { + UserSchema schema = AuditLogService.getAuditLogSchema(user, HttpView.currentContext().getContainer()); + + if (schema != null) + { + TableInfo table = schema.getTable(provider.getEventName(), cf); + TableSelector selector = new TableSelector(table, null, null); + + return selector.getObject(rowId, provider.getEventClass()); + } + } + return null; + } + + @Nullable + public K getAuditEvent(User user, String eventType, int rowId) + { + return getAuditEvent(user, eventType, rowId, null); + } + + public List getAuditEvents(Container container, User user, String eventType, @Nullable SimpleFilter filter, @Nullable Sort sort) + { + return getAuditEvents(container, user, eventType, filter, sort, null); + } + public List getAuditEvents(Container container, User user, String eventType, @Nullable SimpleFilter filter, @Nullable Sort sort, @Nullable ContainerFilter cf) + { + AuditTypeProvider provider = AuditLogService.get().getAuditProvider(eventType); + if (provider != null) + { + UserSchema schema = AuditLogService.getAuditLogSchema(user, container); + + if (schema != null) + { + TableInfo table = schema.getTable(provider.getEventName(), cf); + TableSelector selector = new TableSelector(table, filter, sort); + + return selector.getArrayList(provider.getEventClass()); + } + } + return Collections.emptyList(); + } + + /** + * Ensure that the string properties don't exceed the length of the provisioned columns. + * Values will be trimmed to the max length. + */ + private K validateFields(@NotNull AuditTypeProvider provider, @NotNull K type) + { + ObjectFactory factory = ObjectFactory.Registry.getFactory((Class)type.getClass()); + Map values = new CaseInsensitiveHashMap<>(); + factory.toMap(type, values); + + boolean changed = false; + Domain domain = provider.getDomain(); + + DomainKind domainKind = domain.getDomainKind(); + for (PropertyStorageSpec prop : domainKind.getBaseProperties(domain)) + { + Object value = values.get(prop.getName()); + if (prop.getJdbcType().isText() && value instanceof String s) + { + int scale = prop.getSize(); + if (s.length() > scale) + { + _log.warn("Audit field input : \n" + prop.getName() + "\nexceeded the maximum length : " + scale); + String trimmed = s.substring(0, scale-3) + "..."; + values.put(prop.getName(), trimmed); + changed = true; + } + } + } + + for (DomainProperty dp : domain.getProperties()) + { + // For now, only check for string length like we were doing for the old audit event fields + PropertyDescriptor pd = dp.getPropertyDescriptor(); + Object value = values.get(dp.getName()); + if (pd.isStringType() && value instanceof String s) + { + int scale = dp.getScale(); + if (scale > 0 && s.length() > scale) + { + _log.warn("Audit field input : \n" + pd.getName() + "\nexceeded the maximum length : " + scale); + String trimmed; + if (scale > 100) + trimmed = s.substring(0, scale-3) + "..."; + else + trimmed = s.substring(0, scale); + values.put(pd.getName(), trimmed); + changed = true; + } + } + } + + if (changed) + return factory.fromMap(values); + else + return type; + } +} From 5a43de1fc2522b477385e5708fa0f19fe1de4ec2 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Tue, 7 Oct 2025 08:21:07 -0700 Subject: [PATCH 3/3] appendNowTimestamp --- .../org/labkey/api/data/BaseColumnInfo.java | 45 ++----------------- api/src/org/labkey/api/data/SQLFragment.java | 5 +++ .../org/labkey/api/data/StatementUtils.java | 5 +-- 3 files changed, 11 insertions(+), 44 deletions(-) diff --git a/api/src/org/labkey/api/data/BaseColumnInfo.java b/api/src/org/labkey/api/data/BaseColumnInfo.java index f1a1d08a00a..b9bf8250798 100644 --- a/api/src/org/labkey/api/data/BaseColumnInfo.java +++ b/api/src/org/labkey/api/data/BaseColumnInfo.java @@ -158,7 +158,6 @@ public BaseColumnInfo(ColumnInfo from) this(from, from.getParentTable()); } - public BaseColumnInfo(ColumnInfo from, TableInfo parent) { this(from.getFieldKey(), parent, from.getJdbcType()); @@ -199,7 +198,6 @@ public BaseColumnInfo(ResultSetMetaData rsmd, int col) throws SQLException setAlias(SqlDialect.makeDatabaseIdentifier(rsmd.getColumnName(col), new SQLFragment(rsmd.getColumnName(col)))); } - /* Most ColumnInfos represent a column in the database. However, some are created only for meta-data purposes. * e.g. for DataLoader or "fake" ResultsImpl. * These columns do not have a SqlDialect. This constructor method is useful in that case. In particular it will @@ -235,7 +233,6 @@ public SQLFragment getSql() } } - /* used by TableInfo.addColumn */ public boolean lockName() { @@ -243,7 +240,6 @@ public boolean lockName() return true; } - /** use setFieldKey() avoid ambiguity when columns have "/" */ public void setName(@NotNull String name) { @@ -255,7 +251,6 @@ public void setName(@NotNull String name) _fieldKey = newFieldKey; } - @Override @NotNull public String getName() { @@ -265,7 +260,6 @@ public String getName() return _fieldKey.toString(); } - @Override public void setFieldKey(@NotNull FieldKey key) { @@ -273,14 +267,12 @@ public void setFieldKey(@NotNull FieldKey key) _fieldKey = Objects.requireNonNull(key); } - @Override @NotNull public FieldKey getFieldKey() { return _fieldKey; } - // use only for debugging, will change after call to getAlias() @Override public boolean isAliasSet() @@ -363,7 +355,6 @@ public void copyAttributesFrom(ColumnInfo col) setUserEditable(col.isUserEditable()); //This can impact UniqueId fields if not set } - /* * copy "non-core" attributes, e.g. leave key and type information alone */ @@ -538,7 +529,6 @@ public void setExtraAttributesFrom(BaseColumnInfo col) setScannable(col.isScannable()); } - /** * copy the url string expression from col with the specified rewrites * @param col source of the url StringExpression @@ -571,7 +561,6 @@ public void copyURLFrom(ColumnInfo col, @Nullable FieldKey parent, @Nullable Map setOnClick(col.getOnClick()); } - /* only copy if all field keys are in the map */ public void copyURLFromStrict(ColumnInfo col, Map remap) { @@ -698,7 +687,6 @@ public SqlDialect getSqlDialect() return _parentTable.getSqlDialect(); } - // Return the actual value we have stashed; use this when copying attributes, so you don't hard-code label @Override public String getLabelValue() @@ -706,7 +694,6 @@ public String getLabelValue() return _label; } - @Override public String getLabel() { @@ -715,7 +702,6 @@ public String getLabel() return _label; } - @Override public boolean isFormatStringSet() { @@ -843,14 +829,12 @@ public TableDescription getFkTableDescription() return getFk().getLookupTableDescription(); } - @Override public boolean isUserEditable() { return _isUserEditable; } - @Override public void setUserEditable(boolean editable) { @@ -858,7 +842,6 @@ public void setUserEditable(boolean editable) _isUserEditable = editable; } - @Override public void setDisplayColumnFactory(DisplayColumnFactory factory) { @@ -898,25 +881,23 @@ public boolean isVersionColumn() return JdbcType.BINARY == getJdbcType() && 8 == getScale() && "timestamp".equals(getSqlTypeName()); } - @Override public SQLFragment getVersionUpdateExpression() { if (JdbcType.TIMESTAMP == getJdbcType()) - { - return new SQLFragment().appendValue(new SQLFragment.NowTimestamp()); - } - else if ("_ts".equalsIgnoreCase(getName()) && !getSqlDialect().isSqlServer() && JdbcType.BIGINT == getJdbcType()) + return new SQLFragment().appendNowTimestamp(); + + if (JdbcType.BIGINT == getJdbcType() && "_ts".equalsIgnoreCase(getName()) && !getSqlDialect().isSqlServer()) { TableInfo t = getParentTable(); String tsName = t.getSchema().getName() + "." + Objects.requireNonNull(t.getMetaDataIdentifier()).getId() + "_ts"; String sqlString = getSqlDialect().getStringHandler().quoteStringLiteral(tsName); return new SQLFragment("nextval(" + sqlString + ")"); } + return null; } - @Override public String getInputType() { @@ -936,7 +917,6 @@ else if (getJdbcType() == JdbcType.BOOLEAN) return _inputType; } - @Override public int getInputLength() { @@ -951,7 +931,6 @@ public int getInputLength() return _inputLength; } - @Override public int getInputRows() { @@ -1094,7 +1073,6 @@ public void copyToXml(ColumnType xmlCol, boolean full) } } - public void loadFromXml(ColumnType xmlCol, boolean merge) { checkLocked(); @@ -1448,7 +1426,6 @@ public static boolean booleanFromString(String str) } } - public static boolean booleanFromObj(Object o) { if (null == o) @@ -1657,7 +1634,6 @@ public DisplayColumn getRenderer() } } - public static Collection createFromDatabaseMetaData(String schemaName, SchemaTableInfo parentTable, @Nullable String columnNamePattern) throws SQLException { //Use linked hash map to preserve ordering... @@ -1801,7 +1777,6 @@ else if (key.pkColumnNames.size() == 2 && "container".equalsIgnoreCase(key.fkCol return colMap.values(); } - private static void inferMetadata(BaseColumnInfo col) { String colName = col.getName(); @@ -1856,7 +1831,6 @@ private static void inferMetadata(BaseColumnInfo col) } } - @Override public String getSqlTypeName() { @@ -1874,8 +1848,6 @@ public String getSqlTypeName() return _sqlTypeName; } - - @Override public void setSqlTypeName(String sqlTypeName) { @@ -1908,7 +1880,6 @@ public void setJdbcType(JdbcType type) _sqlTypeName = null; } - @Override public @NotNull JdbcType getJdbcType() { @@ -1932,7 +1903,6 @@ public void setJdbcType(JdbcType type) return _jdbcType == null ? JdbcType.OTHER : _jdbcType; } - @Override public ForeignKey getFk() { @@ -1960,7 +1930,6 @@ public void setFk(@NotNull Builder b) _fk = b.build(); } - @Override public void setScale(int scale) { @@ -1975,7 +1944,6 @@ public void setPrecision(int precision) super.setPrecision(precision); } - /** @return whether the column is part of the primary key for the table */ @Override public boolean isKeyField() @@ -1983,7 +1951,6 @@ public boolean isKeyField() return _isKeyField; } - @Override public void setKeyField(boolean keyField) { @@ -2053,14 +2020,12 @@ public void setIsUnselectable(boolean b) _isUnselectable = b; } - @Override public TableInfo getParentTable() { return _parentTable; } - @Override public void setParentTable(TableInfo parentTable) { @@ -2182,7 +2147,6 @@ public void setValidators(List validators) _validators = copyFixedList(validators); } - @Override public void checkLocked() { @@ -2219,7 +2183,6 @@ public void setCalculated(boolean calculated) _calculated = calculated; } - // If true, you can't use this column when auto-generating LabKey SQL, it is not selected in the underlying query // only query can set this true @Override diff --git a/api/src/org/labkey/api/data/SQLFragment.java b/api/src/org/labkey/api/data/SQLFragment.java index be99434541e..744d025f7db 100644 --- a/api/src/org/labkey/api/data/SQLFragment.java +++ b/api/src/org/labkey/api/data/SQLFragment.java @@ -607,6 +607,11 @@ else if (Double.isFinite(N.doubleValue())) return this; } + public final SQLFragment appendNowTimestamp() + { + return appendValue(new NowTimestamp()); + } + // Issue 27534: Stop using {fn now()} in function declarations // Issue 48864: Query Table's use of web server time can cause discrepancies in created/modified timestamps public final SQLFragment appendValue(NowTimestamp now) diff --git a/api/src/org/labkey/api/data/StatementUtils.java b/api/src/org/labkey/api/data/StatementUtils.java index 3fa83e62ede..580a3432ad1 100644 --- a/api/src/org/labkey/api/data/StatementUtils.java +++ b/api/src/org/labkey/api/data/StatementUtils.java @@ -29,7 +29,6 @@ import org.labkey.api.collections.Sets; import org.labkey.api.data.dialect.MockSqlDialect; import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.dataiterator.SimpleTranslator; import org.labkey.api.dataiterator.TableInsertUpdateDataIterator; import org.labkey.api.exp.MvColumn; import org.labkey.api.exp.PropertyType; @@ -674,7 +673,7 @@ public ParameterMapStatement createStatement(Connection conn, @Nullable Containe if (null != col) { cols.add(col); - values.add(new SQLFragment().appendValue(new SQLFragment.NowTimestamp())); + values.add(new SQLFragment().appendNowTimestamp()); done.add("Created"); } } @@ -691,7 +690,7 @@ public ParameterMapStatement createStatement(Connection conn, @Nullable Containe if (_updateBuiltInColumns && null != colModified) { cols.add(colModified); - values.add(new SQLFragment().appendValue(new SQLFragment.NowTimestamp())); + values.add(new SQLFragment().appendNowTimestamp()); done.add("Modified"); } ColumnInfo colVersion = table.getVersionColumn();