From bf48e663bf5c3c7690669346b7adbebad7fe7c99 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Mon, 17 Nov 2025 15:39:10 -0800 Subject: [PATCH 1/6] FROM EXPANCESTORSOF checkpoint --- .../src/org/labkey/query/sql/QIdentifier.java | 6 +- .../org/labkey/query/sql/QueryLineage.java | 238 ++++++++++++++++++ .../src/org/labkey/query/sql/QuerySelect.java | 31 ++- query/src/org/labkey/query/sql/SqlBase.g | 10 +- query/src/org/labkey/query/sql/SqlParser.java | 138 +++++----- 5 files changed, 357 insertions(+), 66 deletions(-) create mode 100644 query/src/org/labkey/query/sql/QueryLineage.java diff --git a/query/src/org/labkey/query/sql/QIdentifier.java b/query/src/org/labkey/query/sql/QIdentifier.java index beaecf72633..34500e87c3c 100644 --- a/query/src/org/labkey/query/sql/QIdentifier.java +++ b/query/src/org/labkey/query/sql/QIdentifier.java @@ -78,9 +78,9 @@ public FieldKey getFieldKey() public String getIdentifier() { - if (getTokenType() == SqlBaseParser.IDENT) - return getTokenText(); - return LabKeySql.unquoteIdentifier(getTokenText()); + if (getTokenType() == SqlBaseParser.QUOTED_IDENTIFIER) + return LabKeySql.unquoteIdentifier(getTokenText()); + return getTokenText(); } diff --git a/query/src/org/labkey/query/sql/QueryLineage.java b/query/src/org/labkey/query/sql/QueryLineage.java new file mode 100644 index 00000000000..cd48dcfd344 --- /dev/null +++ b/query/src/org/labkey/query/sql/QueryLineage.java @@ -0,0 +1,238 @@ +package org.labkey.query.sql; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.BaseColumnInfo; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.exp.api.ExpLineageOptions; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.query.FieldKey; +import org.labkey.data.xml.ColumnType; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public class QueryLineage extends AbstractQueryRelation +{ + final QuerySelect sourceSelect; + final boolean ancestors; + + QueryLineage(Query query, QuerySelect source, String alias, boolean ancestors) + { + super(query, query.getSchema(), Objects.toString(alias, "explineage_" + query.incrementAliasCounter())); + this.sourceSelect = source; + this.ancestors = ancestors; + } + +// @Override +// public String getAlias() +// { +// return sourceSelect.getAlias(); +// } + + @Override + public void declareFields() + { + sourceSelect.declareFields(); + } + + @Override + public void resolveFields() + { + sourceSelect.resolveFields(); + } + + @Override + public TableInfo getTableInfo() + { + throw new UnsupportedOperationException(); + } + + @Override + public Map getAllColumns() + { + return Map.of( + "Depth", Objects.requireNonNull(getColumn("depth")), + "FromObject", Objects.requireNonNull(getColumn("fromobject")), + "ToObject", Objects.requireNonNull(getColumn("toobject"))); + } + + @Override + public @Nullable AbstractQueryRelation.RelationColumn getFirstColumn() + { + return getColumn("fromobject"); + } + + @Override + public @Nullable AbstractQueryRelation.RelationColumn getColumn(@NotNull String name) + { + return switch (name.toLowerCase()) + { + case "depth", "fromobject", "toobject" -> new LineageColumn(name.toLowerCase()); + default -> null; + }; + } + + @Override + public int getSelectedColumnCount() + { + return 3; + } + + @Override + public @Nullable AbstractQueryRelation.RelationColumn getLookupColumn(@NotNull RelationColumn parent, @NotNull String name) + { + throw new UnsupportedOperationException(); + } + + @Override + public @Nullable AbstractQueryRelation.RelationColumn getLookupColumn(@NotNull RelationColumn parent, ColumnType.@NotNull Fk fk, @NotNull String name) + { + throw new UnsupportedOperationException(); + } + + @Override + public SQLFragment getSql() + { + throw new UnsupportedOperationException(); + } + + @Override + public SQLFragment getFromSql() + { + SqlDialect d = _query.getSchema().getDbSchema().getSqlDialect(); + + ExpLineageOptions options = new ExpLineageOptions(ancestors, !ancestors, 1000); + options.setForLookup(true); // remove intermediate edges + options.setUseObjectIds(true); // expObject() returns objectid not lsid + + SQLFragment sql = new SQLFragment(); + sql.appendComment("", d); + // CONSIDER: use CTE for sourceSelect.getSql() + SQLFragment lineageSql = ExperimentService.get().generateExperimentTreeSQL(sourceSelect.getSql(), options); + sql.append("(\n").append(lineageSql).append("\n) AS ").appendIdentifier(getAlias()); + sql.appendComment("", d); + return sql; + } + + @Override + public String getQueryText() + { + return (ancestors?"EXPANCESTORSOF":"EXPDESCENDANTSOF") + "(" + sourceSelect.getQueryText() + ")"; + } + + @Override + public void setContainerFilter(ContainerFilter containerFilter) + { + sourceSelect.setContainerFilter(containerFilter); + } + + @Override + public Set getSuggestedColumns(Set selected) + { + return Set.of(); + } + + + private class LineageColumn extends RelationColumn + { + final FieldKey _fieldKey; + final String _alias; + final JdbcType _jdbcType; + + LineageColumn(String name) + { + String alias; + FieldKey fk; + JdbcType jdbcType = JdbcType.BIGINT; + switch (name.toLowerCase()) + { + case "depth": + alias = "depth"; + fk = new FieldKey(null, "Depth"); + jdbcType = JdbcType.INTEGER; + break; + case "fromobject": + alias = "self"; + fk = new FieldKey(null, "FromObject"); + break; + case "toobject": + alias = "objectid"; + fk = new FieldKey(null, "ToObject"); + break; + default: + throw new IllegalArgumentException("Unknown column name: " + name); + } + _fieldKey = fk; + _alias = alias; + _jdbcType = jdbcType; + } + + @Override + SQLFragment getInternalSql() + { + return new SQLFragment().appendDottedIdentifiers(QueryLineage.this.getAlias(), getAlias()); + } + + @Override + void copyColumnAttributesTo(@NotNull BaseColumnInfo to) + { + to.setJdbcType(_jdbcType); + } + + @Override + public FieldKey getFieldKey() + { + return _fieldKey; + } + + @Override + String getAlias() + { + return _alias; + } + + @Override + AbstractQueryRelation getTable() + { + return QueryLineage.this; + } + + @Override + boolean isHidden() + { + return false; + } + + @Override + String getPrincipalConceptCode() + { + return ""; + } + + @Override + String getConceptURI() + { + return ""; + } + + @Override + public @NotNull JdbcType getJdbcType() + { + return _jdbcType; + } + + @Override + public Collection gatherInvolvedSelectColumns(Collection collect) + { + return List.of(); + } + } +} diff --git a/query/src/org/labkey/query/sql/QuerySelect.java b/query/src/org/labkey/query/sql/QuerySelect.java index 95ecbe024a7..fdd663a4d05 100644 --- a/query/src/org/labkey/query/sql/QuerySelect.java +++ b/query/src/org/labkey/query/sql/QuerySelect.java @@ -647,8 +647,16 @@ private QJoinOrTable parseNode(QNode r) { if (r.getTokenType() == SqlBaseParser.RANGE) { - if (r.getFirstChild().getTokenType() == SqlBaseParser.VALUES) + var table = r.getFirstChild(); + var childType = table.getTokenType(); + if (childType == SqlBaseParser.VALUES) + { return parseValues(r); + } + if (childType == SqlBaseParser.METHOD_CALL) + { + return parseLineage(r); + } return parseRange(r); } else if (r.getTokenType() == SqlBaseParser.JOIN) @@ -660,6 +668,27 @@ else if (r.getTokenType() == SqlBaseParser.JOIN) } } + private QTable parseLineage(final QNode range) + { + QMethodCall methodCall = (QMethodCall)range.childList().get(0); + int methodType = methodCall.childList().get(0).getTokenType(); + QQuery subQuery = (QQuery)methodCall.childList().get(1); + QIdentifier alias = (QIdentifier)range.childList().get(1); + QuerySelect sourceQuery = new QuerySelect(_query, subQuery, true); + QueryLineage lineageQuery = new QueryLineage(_query, sourceQuery, alias.getIdentifier(), methodType==SqlBaseParser.EXPANCESTORSOF); + QTable methodTable = new QTable(lineageQuery, lineageQuery.getAlias()); + FieldKey aliasKey = methodTable.getAlias(); + if (_tables.containsKey(aliasKey)) + { + parseError(aliasKey + " was specified more than once", alias); + } + else + { + _tables.put(aliasKey, methodTable); + } + return methodTable; + } + private QTable parseValues(QNode node) { int countChildren = node.childList().size(); diff --git a/query/src/org/labkey/query/sql/SqlBase.g b/query/src/org/labkey/query/sql/SqlBase.g index 922c1e3dedc..2db0a05793d 100644 --- a/query/src/org/labkey/query/sql/SqlBase.g +++ b/query/src/org/labkey/query/sql/SqlBase.g @@ -416,7 +416,13 @@ fromRange ( (subQuery) => subQuery CLOSE (AS? identifier)? -> ^(RANGE subQuery identifier?) | joinExpression CLOSE -> joinExpression ) - ; + | tableMethod (AS? identifier)? -> ^(RANGE tableMethod identifier?) + ; + + +tableMethod + : (EXPANCESTORSOF | EXPDESCENANTSOF) op=OPEN^ {$op.setType(METHOD_CALL);} subQuery CLOSE! + ; tableSpecificationWithAnnotation @@ -671,7 +677,7 @@ likeEscape inList - : (EXPDESCENDANTSOF|EXPANCESTORSOF)^ OPEN! subQuery CLOSE! + : (EXPANCESTORSOF | EXPDESCENANTSOF) op=OPEN^ {$op.setType(METHOD_CALL);} subQuery CLOSE! | compoundExpr -> ^(IN_LIST compoundExpr) ; diff --git a/query/src/org/labkey/query/sql/SqlParser.java b/query/src/org/labkey/query/sql/SqlParser.java index f8d70874a22..6bc029ccd7e 100644 --- a/query/src/org/labkey/query/sql/SqlParser.java +++ b/query/src/org/labkey/query/sql/SqlParser.java @@ -76,6 +76,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import static org.labkey.query.sql.antlr.SqlBaseParser.*; @@ -83,10 +84,8 @@ /** * SqlParser is responsible for the first two phases of the SQL transformation process - * * step one - the ANTLR parser returns a tree of Nodes * step two - translate the tree into a tree of QNodes - * */ @SuppressWarnings({"ThrowableResultOfMethodCallIgnored","ThrowableInstanceNeverThrown"}) @@ -94,7 +93,6 @@ public class SqlParser { // these are not a regular method and need special handling public static final String FIND_COLUMN_METHOD_NAME = "findcolumn"; - public static final String IFDEFINED_METHOD_NAME = "ifdefined"; private static final Logger _log = LogHelper.getLogger(SqlParser.class, "LabKey SQL parser"); @@ -241,11 +239,10 @@ else if (null != warnings) { CommonTree parseRoot = (CommonTree) selectScope.getTree(); assert parseRoot != null; -// assert dump(parseRoot); assert parseRoot.getType() == STATEMENT; - assert parseRoot.getChildCount()==1 || parseRoot.getChildCount()==2 || parseRoot.getChildCount() == 3; + assert parseRoot.getChildCount() == 1 || parseRoot.getChildCount() == 2 || parseRoot.getChildCount() == 3; - ArrayList list = new ArrayList<>((Collection)parseRoot.getChildren()); + ArrayList list = new ArrayList<>((Collection) parseRoot.getChildren()); CommonTree parameters; @@ -316,12 +313,17 @@ else if (null != warnings) else errors.add(new QueryParseException("This does not look like a WITH, SELECT or UNION query", null, 0, 0)); } - + for (Throwable e : _parseErrors) { errors.add(wrapParseException(e)); } + if (null != _root) + { + dump(_root); + _log.debug(toPrefixString(_root)); + } return _root; } catch (Exception e) @@ -398,7 +400,7 @@ public QExpr parseExpr(String str, boolean constExpression, List l = new LinkedList<>(); for (int i=0 ; i(List.of(lhs, rhs.childList().get(1)))); + return qInLineage; + } + } case METHOD_CALL: { - QNode id = first(children), exprList = second(children); + @NotNull QNode id = firstOrThrow(children); + @NotNull QNode exprList = secondOrThrow(children); // check for special case table method "findColumn", this isn't a real method so it's easier if it has its own node type @@ -964,26 +965,31 @@ else if (name.equals("age")) if (args.size() == 3) validateTimestampConstant(args.get(2)); } - - try + + // special case for table returning method + var isTableResultMethod = id.getTokenType() == EXPANCESTORSOF || id.getTokenType() == EXPDESCENANTSOF; + if (!isTableResultMethod) { - Method m = Method.resolve(_dialect, name); - if (null != m) + try { - m.validate(node, exprList.childList(), _parseErrors, _parseWarnings); + Method m = Method.resolve(_dialect, name); + if (null != m) + { + m.validate(node, exprList.childList(), _parseErrors, _parseWarnings); + } + } + catch (IllegalArgumentException x) + { + if (failOnUnrecognizedMethodName) + _parseErrors.add(new QueryParseException("Unknown method " + name, null, id.getLine(), id.getColumn())); } } - catch (IllegalArgumentException x) - { - if (failOnUnrecognizedMethodName) - _parseErrors.add(new QueryParseException("Unknown method " + name, null, id.getLine(), id.getColumn())); - } - break; } case AGGREGATE: { - if (constExpr) return constError(node); + if (constExpr) + return constError(node); QAggregate qAggregate = (QAggregate)qnode(node, children, false); if (!qAggregate.getType().dialectSupports(_dialect)) { @@ -1009,7 +1015,7 @@ else if (name.equals("age")) case TIMESTAMP_LITERAL: case DATE_LITERAL: { - String s = LabKeySql.unquoteString(first(children).getTokenText()); + String s = LabKeySql.unquoteString(firstOrThrow(children).getTokenText()); try { if (node.getType() == TIMESTAMP_LITERAL) @@ -1064,12 +1070,12 @@ else if (name.equals("age")) } case RANGE: { - if (constExpr) return constError(node); + if (constExpr) + return constError(node); // copy an annotations on the table specifications to the range node QUnknownNode range = (QUnknownNode)qnode(node, children, false); var annotations = ((SupportsAnnotations)node.getChild(0)).getAnnotations(); - if (null != annotations) - range.setAnnotations(QNode.convertAnnotations(annotations)); + range.setAnnotations(QNode.convertAnnotations(annotations)); return range; } default: @@ -1161,6 +1167,8 @@ private QSelect transformSelect(QSelect select, QSelectFrom innerSelectFrom, // [out] inner SelectFrom to be filled in @NotNull Map groupByAliasMap) // [out] alias map for GroupBy { + if (null == _dialect) + throw new IllegalStateException("dialect is required"); AliasManager aliasManager = new AliasManager(_dialect); // Need to assign unique names to selected fields for them to be used in outer select for (QNode child : select.children()) // Claim existing aliases { @@ -1396,7 +1404,7 @@ private QFieldKey substitutePath(String pathString) _parseErrors.add(new QueryParseException("Path substition is empty", null, -1, -1)); return null; } - // NOTE the "/" forces this to be interpreted as directory (not a schema name) + // NOTE the "/" forces this to be interpreted as a directory (not a schema name) if (!pathString.endsWith(("/"))) pathString += "/"; return new QIdentifier(pathString); @@ -1510,12 +1518,20 @@ private static QNode first(LinkedList children) return !children.isEmpty() ? children.get(0) : null; } + private static @NotNull QNode firstOrThrow(LinkedList children) + { + return Objects.requireNonNull(first(children)); + } private static QNode second(LinkedList children) { return children.size() > 1 ? children.get(1) : null; } - + + private static @NotNull QNode secondOrThrow(LinkedList children) + { + return Objects.requireNonNull(second(children)); + } private QNode constantToStringNode(QNode node) { @@ -1640,6 +1656,8 @@ QNode qnode(CommonTree node, boolean constExpr) case AS: q = new QAs(); break; + case EXPANCESTORSOF: + case EXPDESCENDANTSOF: case IDENT: case QUOTED_IDENTIFIER: return QIdentifier.create(node); @@ -1767,7 +1785,6 @@ QNode qnode(CommonTree node, boolean constExpr) case NOT: case AND: case OR: case LIKE: case NOT_LIKE: case BIT_AND: case BIT_OR: case BIT_XOR: case UNARY_PLUS: Operator op = Operator.ofTokenType(type); - assert op != null : "No Operation found for type " + type + ". If you are on a development system, you may need to rebuild"; if (op == null) { _parseErrors.add(new QueryParseException("Unexpected token '" + node.getText() + "'", null, node.getLine(), node.getCharPositionInLine())); @@ -1777,10 +1794,10 @@ QNode qnode(CommonTree node, boolean constExpr) break; case IN: case NOT_IN: CommonTree right = (CommonTree)node.getChild(1); - if (right.getToken().getType() == EXPANCESTORSOF || right.getToken().getType() == EXPDESCENDANTSOF) + if (right.getToken().getType() == METHOD_CALL) { - node.setChild(1, right.getChild(0)); - q = new QInLineage(type==IN, right.getToken().getType() == EXPANCESTORSOF); + // We should have handled this in convertTree() + throw new QueryParseException("Error parsing IN expression", null, node.getLine(), node.getCharPositionInLine()); } else { @@ -1965,6 +1982,7 @@ class delete elements fetch indices insert into limit new set update versioned b + @SuppressWarnings("JUnitMalformedDeclaration") public static class SqlParserTestCase extends Assert { List> parseExprs = Arrays.asList( @@ -2071,7 +2089,7 @@ public static class SqlParserTestCase extends Assert "(WithQuery (WITH (AS peeps1 (QUERY (SELECT_FROM (SELECT ROW_STAR) (FROM (RANGE R))))) (AS peeps (UNION (QUERY (SELECT_FROM (SELECT ROW_STAR) (FROM (RANGE peeps1)))) (QUERY (SELECT_FROM (SELECT ROW_STAR) (FROM (RANGE peeps))) (WHERE (= 1 0)))))) (QUERY (SELECT_FROM (SELECT ROW_STAR) (FROM (RANGE peeps)))))") ); - private void good(String sql) throws Exception + private void good(String sql) { List errors = new ArrayList<>(); QNode q = (new SqlParser()).parseQuery(sql,errors,null); @@ -2091,7 +2109,7 @@ private void fail(QueryParseException qpe, String sql) private void bad(String sql) { List errors = new ArrayList<>(); - QNode q = (new SqlParser()).parseQuery(sql,errors,null); + (new SqlParser()).parseQuery(sql,errors,null); if (errors.isEmpty()) fail("BAD: " + sql); } @@ -2175,7 +2193,7 @@ public void testSql() } } long end = System.currentTimeMillis(); - _log.trace("SqlParser.testSql(): " + DateUtil.formatDuration(end-start)); + _log.trace("SqlParser.testSql(): {}", DateUtil.formatDuration(end - start)); } @Test From 9804b226df982be4c63dafae7363e3db83071efd Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Tue, 18 Nov 2025 11:54:48 -0800 Subject: [PATCH 2/6] Some validation --- .../org/labkey/query/sql/QueryLineage.java | 26 ++++++++++++++----- .../src/org/labkey/query/sql/QuerySelect.java | 5 ++-- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/query/src/org/labkey/query/sql/QueryLineage.java b/query/src/org/labkey/query/sql/QueryLineage.java index cd48dcfd344..895dbda41ba 100644 --- a/query/src/org/labkey/query/sql/QueryLineage.java +++ b/query/src/org/labkey/query/sql/QueryLineage.java @@ -1,5 +1,6 @@ package org.labkey.query.sql; +import org.apache.commons.lang3.Strings; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.data.BaseColumnInfo; @@ -11,6 +12,8 @@ import org.labkey.api.exp.api.ExpLineageOptions; import org.labkey.api.exp.api.ExperimentService; import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryParseException; +import org.labkey.api.query.column.BuiltInColumnTypes; import org.labkey.data.xml.ColumnType; import java.util.Collection; @@ -24,18 +27,27 @@ public class QueryLineage extends AbstractQueryRelation final QuerySelect sourceSelect; final boolean ancestors; - QueryLineage(Query query, QuerySelect source, String alias, boolean ancestors) + QueryLineage(Query query, QNode token, QuerySelect source, String alias, boolean ancestors) { super(query, query.getSchema(), Objects.toString(alias, "explineage_" + query.incrementAliasCounter())); this.sourceSelect = source; this.ancestors = ancestors; - } -// @Override -// public String getAlias() -// { -// return sourceSelect.getAlias(); -// } + sourceSelect.setSkipSuggestedColumns(true); + var all = source.getAllColumns(); + if (1 != all.size()) + { + query.getParseErrors().add(new QueryParseException(token.getTokenText() + " subquery must have one column", null, token.getLine(), token.getColumn())); + } + else + { + var relationColumn = all.values().iterator().next(); + var col = new BaseColumnInfo("?"); + relationColumn.copyColumnAttributesTo(col); + if (!BuiltInColumnTypes.EXPOBJECTID_CONCEPT_URI.equals(col.getConceptURI())) + query.getParseErrors().add(new QueryParseException(token.getTokenText() + " requires an object column", null, token.getLine(), token.getColumn())); + } + } @Override public void declareFields() diff --git a/query/src/org/labkey/query/sql/QuerySelect.java b/query/src/org/labkey/query/sql/QuerySelect.java index fdd663a4d05..ce421adff82 100644 --- a/query/src/org/labkey/query/sql/QuerySelect.java +++ b/query/src/org/labkey/query/sql/QuerySelect.java @@ -671,11 +671,12 @@ else if (r.getTokenType() == SqlBaseParser.JOIN) private QTable parseLineage(final QNode range) { QMethodCall methodCall = (QMethodCall)range.childList().get(0); - int methodType = methodCall.childList().get(0).getTokenType(); + var methodIdentifier = methodCall.childList().get(0); + int methodType = methodIdentifier.getTokenType(); QQuery subQuery = (QQuery)methodCall.childList().get(1); QIdentifier alias = (QIdentifier)range.childList().get(1); QuerySelect sourceQuery = new QuerySelect(_query, subQuery, true); - QueryLineage lineageQuery = new QueryLineage(_query, sourceQuery, alias.getIdentifier(), methodType==SqlBaseParser.EXPANCESTORSOF); + QueryLineage lineageQuery = new QueryLineage(_query, methodIdentifier, sourceQuery, alias.getIdentifier(), methodType==SqlBaseParser.EXPANCESTORSOF); QTable methodTable = new QTable(lineageQuery, lineageQuery.getAlias()); FieldKey aliasKey = methodTable.getAlias(); if (_tables.containsKey(aliasKey)) From af8ea1c059b9c6a25739e8d2784e99f8f077ef0e Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Tue, 18 Nov 2025 13:23:34 -0800 Subject: [PATCH 3/6] ExpObjectDataColumn --- .../org/labkey/api/exp/query/ExpTable.java | 35 +++++++++++++++++++ .../labkey/experiment/api/ExpTableImpl.java | 29 +-------------- .../org/labkey/query/sql/QueryLineage.java | 6 ++++ query/src/org/labkey/query/sql/SqlBase.g | 2 +- 4 files changed, 43 insertions(+), 29 deletions(-) diff --git a/api/src/org/labkey/api/exp/query/ExpTable.java b/api/src/org/labkey/api/exp/query/ExpTable.java index 3da8597280c..9d81d735dbe 100644 --- a/api/src/org/labkey/api/exp/query/ExpTable.java +++ b/api/src/org/labkey/api/exp/query/ExpTable.java @@ -22,7 +22,9 @@ import org.labkey.api.data.Container; import org.labkey.api.data.ContainerFilter; import org.labkey.api.data.ContainerFilterable; +import org.labkey.api.data.DataColumn; import org.labkey.api.data.MutableColumnInfo; +import org.labkey.api.data.RenderContext; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.TableInfo; import org.labkey.api.dataiterator.DataIteratorContext; @@ -134,4 +136,37 @@ default ColumnInfo getExpObjectColumn() { return null; } + + class ExpObjectDataColumn extends DataColumn + { + public ExpObjectDataColumn(ColumnInfo colInfo) + { + super(colInfo); + } + + @Override + public Object getValue(RenderContext ctx) + { + return 0; + } + + @Override + public Object getDisplayValue(RenderContext ctx) + { + var v = super.getValue(ctx); + return null == v ? v : "lineage object"; + } + + @Override + public boolean isSortable() + { + return false; + } + + @Override + public boolean isFilterable() + { + return false; + } + } } diff --git a/experiment/src/org/labkey/experiment/api/ExpTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpTableImpl.java index 6f2ca2997b7..8d73a6b178e 100644 --- a/experiment/src/org/labkey/experiment/api/ExpTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpTableImpl.java @@ -513,38 +513,11 @@ public SQLFragment getSQL(String tableAlias, DbSchema schema, SQLFragment[] argu @Override public MutableColumnInfo createColumnInfo(TableInfo parentTable, ColumnInfo[] arguments, String alias) { - var objectColumn = getExpObjectColumn(); if (null == _expObjectColumnName) return new NullColumnInfo(parentTable, "_exptable_object_", JdbcType.INTEGER); var ret = super.createColumnInfo(parentTable, arguments, "_exptable_object_"); ret.setConceptURI(BuiltInColumnTypes.EXPOBJECTID_CONCEPT_URI); - ret.setDisplayColumnFactory(colInfo -> new DataColumn(colInfo) - { - @Override - public Object getValue(RenderContext ctx) - { - return 0; - } - - @Override - public Object getDisplayValue(RenderContext ctx) - { - var v = super.getValue(ctx); - return null == v ? v : "lineage object"; - } - - @Override - public boolean isSortable() - { - return false; - } - - @Override - public boolean isFilterable() - { - return false; - } - }); + ret.setDisplayColumnFactory(ExpObjectDataColumn::new); return ret; } } diff --git a/query/src/org/labkey/query/sql/QueryLineage.java b/query/src/org/labkey/query/sql/QueryLineage.java index 895dbda41ba..706fa516610 100644 --- a/query/src/org/labkey/query/sql/QueryLineage.java +++ b/query/src/org/labkey/query/sql/QueryLineage.java @@ -11,6 +11,7 @@ import org.labkey.api.data.dialect.SqlDialect; import org.labkey.api.exp.api.ExpLineageOptions; import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.query.ExpTable; import org.labkey.api.query.FieldKey; import org.labkey.api.query.QueryParseException; import org.labkey.api.query.column.BuiltInColumnTypes; @@ -197,6 +198,11 @@ SQLFragment getInternalSql() void copyColumnAttributesTo(@NotNull BaseColumnInfo to) { to.setJdbcType(_jdbcType); + if (!"depth".equals(_alias)) + { + to.setConceptURI(BuiltInColumnTypes.EXPOBJECTID_CONCEPT_URI); + to.setDisplayColumnFactory(ExpTable.ExpObjectDataColumn::new); + } } @Override diff --git a/query/src/org/labkey/query/sql/SqlBase.g b/query/src/org/labkey/query/sql/SqlBase.g index 2db0a05793d..2b1598bdefe 100644 --- a/query/src/org/labkey/query/sql/SqlBase.g +++ b/query/src/org/labkey/query/sql/SqlBase.g @@ -421,7 +421,7 @@ fromRange tableMethod - : (EXPANCESTORSOF | EXPDESCENANTSOF) op=OPEN^ {$op.setType(METHOD_CALL);} subQuery CLOSE! + : (EXPANCESTORSOF | EXPDESCENDANTSOF) op=OPEN^ {$op.setType(METHOD_CALL);} subQuery CLOSE! ; From 2c620a05b2a9f0e0c451de96adbcb9d907d7521e Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Wed, 19 Nov 2025 13:05:09 -0800 Subject: [PATCH 4/6] case sensitivity fix --- api/src/org/labkey/api/exp/query/ExpSchema.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/org/labkey/api/exp/query/ExpSchema.java b/api/src/org/labkey/api/exp/query/ExpSchema.java index 5fd2cedf774..6eb75058910 100644 --- a/api/src/org/labkey/api/exp/query/ExpSchema.java +++ b/api/src/org/labkey/api/exp/query/ExpSchema.java @@ -16,6 +16,7 @@ package org.labkey.api.exp.query; +import org.apache.commons.lang3.Strings; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.collections.CaseInsensitiveHashSet; @@ -502,11 +503,10 @@ public QuerySchema getSchema(String name) if (_restricted) return null; - // CONSIDER: also support hidden "samples" schema ? - if (name.equals(NestedSchemas.materials.name())) + if (Strings.CI.equals(name, NestedSchemas.materials.name())) return new SamplesSchema(SchemaKey.fromParts(getName(), NestedSchemas.materials.name()), getUser(), getContainer()); - if (name.equals(NestedSchemas.data.name())) + if (Strings.CI.equals(name, NestedSchemas.data.name())) return new DataClassUserSchema(getContainer(), getUser()); return super.getSchema(name); From f478351349e92aab0f73a8939e6a1b6b1803d6a2 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Wed, 19 Nov 2025 16:24:07 -0800 Subject: [PATCH 5/6] Experimental Feature --- api/src/org/labkey/api/exp/api/ExperimentService.java | 2 ++ .../src/org/labkey/experiment/ExperimentModule.java | 2 ++ query/src/org/labkey/query/sql/QuerySelect.java | 8 +++++++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/api/src/org/labkey/api/exp/api/ExperimentService.java b/api/src/org/labkey/api/exp/api/ExperimentService.java index a32fffb5f08..d317517c57b 100644 --- a/api/src/org/labkey/api/exp/api/ExperimentService.java +++ b/api/src/org/labkey/api/exp/api/ExperimentService.java @@ -127,6 +127,8 @@ public interface ExperimentService extends ExperimentRunTypeSource String ILLEGAL_PARENT_ALIAS_CHARSET = "/:<>$[]{};,`\"~!@#$%^*=|?\\"; + String EXPERIMENTAL_FEATURE_FROM_EXPANCESTORS = "org.labkey.api.exp.api.ExperimentService#FROM_EXPANCESTORS"; + int SIMPLE_PROTOCOL_FIRST_STEP_SEQUENCE = 1; int SIMPLE_PROTOCOL_CORE_STEP_SEQUENCE = 10; int SIMPLE_PROTOCOL_EXTRA_STEP_SEQUENCE = 15; diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index 67ab8161805..c55f70faa3b 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -267,6 +267,8 @@ protected void init() } OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.QUANTITY_COLUMN_SUFFIX_TESTING, "Quantity column suffix testing", "If a column name contains a \"__\" suffix, this feature allows for testing it as a Quantity display column", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(ExperimentService.EXPERIMENTAL_FEATURE_FROM_EXPANCESTORS, "SQL syntax: 'FROM EXPANCESTORS()'", + "Support for querying lineage of experiment objects", false); RoleManager.registerPermission(new DesignVocabularyPermission(), true); RoleManager.registerRole(new SampleTypeDesignerRole()); diff --git a/query/src/org/labkey/query/sql/QuerySelect.java b/query/src/org/labkey/query/sql/QuerySelect.java index ce421adff82..c863c3ad90d 100644 --- a/query/src/org/labkey/query/sql/QuerySelect.java +++ b/query/src/org/labkey/query/sql/QuerySelect.java @@ -45,6 +45,7 @@ import org.labkey.api.query.QuerySchema; import org.labkey.api.query.UserSchema; import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.OptionalFeatureService; import org.labkey.api.util.ContainerContext; import org.labkey.api.util.GUID; import org.labkey.api.util.MemTracker; @@ -67,6 +68,7 @@ import java.util.stream.Collectors; import static java.util.Objects.requireNonNull; +import static org.labkey.api.exp.api.ExperimentService.EXPERIMENTAL_FEATURE_FROM_EXPANCESTORS; public class QuerySelect extends AbstractQueryRelation implements Cloneable @@ -679,7 +681,11 @@ private QTable parseLineage(final QNode range) QueryLineage lineageQuery = new QueryLineage(_query, methodIdentifier, sourceQuery, alias.getIdentifier(), methodType==SqlBaseParser.EXPANCESTORSOF); QTable methodTable = new QTable(lineageQuery, lineageQuery.getAlias()); FieldKey aliasKey = methodTable.getAlias(); - if (_tables.containsKey(aliasKey)) + if (!OptionalFeatureService.get().isFeatureEnabled(EXPERIMENTAL_FEATURE_FROM_EXPANCESTORS)) + { + parseError("Syntax error in FROM clause", range); + } + else if (_tables.containsKey(aliasKey)) { parseError(aliasKey + " was specified more than once", alias); } From e027d13b476038a66df3eb1e2869901a971bbfa5 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Thu, 20 Nov 2025 12:56:42 -0800 Subject: [PATCH 6/6] fix typo! --- query/src/org/labkey/query/sql/SqlBase.g | 2 +- query/src/org/labkey/query/sql/SqlParser.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/query/src/org/labkey/query/sql/SqlBase.g b/query/src/org/labkey/query/sql/SqlBase.g index 2b1598bdefe..908ce409aff 100644 --- a/query/src/org/labkey/query/sql/SqlBase.g +++ b/query/src/org/labkey/query/sql/SqlBase.g @@ -677,7 +677,7 @@ likeEscape inList - : (EXPANCESTORSOF | EXPDESCENANTSOF) op=OPEN^ {$op.setType(METHOD_CALL);} subQuery CLOSE! + : (EXPANCESTORSOF | EXPDESCENDANTSOF) op=OPEN^ {$op.setType(METHOD_CALL);} subQuery CLOSE! | compoundExpr -> ^(IN_LIST compoundExpr) ; diff --git a/query/src/org/labkey/query/sql/SqlParser.java b/query/src/org/labkey/query/sql/SqlParser.java index 6bc029ccd7e..9321fb5175c 100644 --- a/query/src/org/labkey/query/sql/SqlParser.java +++ b/query/src/org/labkey/query/sql/SqlParser.java @@ -885,7 +885,7 @@ else if (divisorType==NUM_DOUBLE || divisorType==NUM_FLOAT || divisorType==NUM_I { // rewrite "IN EXPANCESTORS" "IN EXPDESCENDANTS" var method = rhs.getFirstChild(); - if (method.getTokenType() != EXPANCESTORSOF && method.getTokenType() != EXPDESCENANTSOF) + if (method.getTokenType() != EXPANCESTORSOF && method.getTokenType() != EXPDESCENDANTSOF) { _parseErrors.add(new QueryParseException("Illegal syntax near 'IN'", null, node.getLine(), node.getCharPositionInLine())); return null; @@ -967,7 +967,7 @@ else if (name.equals("age")) } // special case for table returning method - var isTableResultMethod = id.getTokenType() == EXPANCESTORSOF || id.getTokenType() == EXPDESCENANTSOF; + var isTableResultMethod = id.getTokenType() == EXPANCESTORSOF || id.getTokenType() == EXPDESCENDANTSOF; if (!isTableResultMethod) { try