From a15f787cc59f40365ef9c41d43b9f7d92fe45720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 12 Jan 2026 14:23:49 +0100 Subject: [PATCH 01/22] add transform context type --- .../sql/bridge/ElasticAggregation.scala | 12 +++- .../elastic/sql/bridge/ElasticBridge.scala | 9 +-- .../elastic/sql/bridge/ElasticCriteria.scala | 4 +- .../elastic/sql/bridge/package.scala | 64 +++++++++++++++---- .../elastic/client/ElasticClientApi.scala | 1 - .../client/ElasticClientDelegator.scala | 11 ++-- .../elastic/client/IndicesApi.scala | 6 +- .../elastic/client/NopeClientApi.scala | 9 ++- .../elastic/client/SearchApi.scala | 6 +- .../sql/bridge/ElasticAggregation.scala | 12 +++- .../elastic/sql/bridge/ElasticBridge.scala | 6 +- .../elastic/sql/bridge/ElasticCriteria.scala | 4 +- .../elastic/sql/bridge/package.scala | 63 ++++++++++++++---- .../elastic/client/jest/JestSearchApi.scala | 8 ++- .../client/rest/RestHighLevelClientApi.scala | 7 +- .../client/rest/RestHighLevelClientApi.scala | 9 +-- .../elastic/client/java/JavaClientApi.scala | 8 ++- .../elastic/client/java/JavaClientApi.scala | 8 ++- sql/src/main/resources/softnetwork-sql.conf | 1 + .../elastic/sql/config/ElasticSqlConfig.scala | 3 +- .../elastic/sql/config/ElasticSqlConfig.scala | 3 +- .../app/softnetwork/elastic/sql/package.scala | 4 +- .../elastic/client/MockElasticClientApi.scala | 8 ++- 23 files changed, 195 insertions(+), 71 deletions(-) diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index 72132c8d..7f0217d4 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.PainlessContext +import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType} import app.softnetwork.elastic.sql.`type`.SQLTemporal import app.softnetwork.elastic.sql.query.{ Asc, @@ -100,7 +100,10 @@ object ElasticAggregation { having: Option[Criteria], bucketsDirection: Map[String, SortOrder], allAggregations: Map[String, SQLAggregation] - )(implicit timestamp: Long): ElasticAggregation = { + )(implicit + timestamp: Long, + contextType: PainlessContextType + ): ElasticAggregation = { import sqlAgg._ val sourceField = identifier.path @@ -348,7 +351,10 @@ object ElasticAggregation { having: Option[Criteria], nested: Option[NestedElement], allElasticAggregations: Seq[ElasticAggregation] - )(implicit timestamp: Long): Seq[Aggregation] = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Seq[Aggregation] = { for (tree <- buckets) yield { val treeNodes = tree.sortBy(_.level).reverse.foldLeft(Seq.empty[NodeAggregation]) { (current, node) => diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala index c013e82c..c3b2893f 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala @@ -16,6 +16,7 @@ package app.softnetwork.elastic.sql.bridge +import app.softnetwork.elastic.sql.PainlessContextType import app.softnetwork.elastic.sql.operator.AND import app.softnetwork.elastic.sql.query.{ BetweenExpr, @@ -36,16 +37,16 @@ import app.softnetwork.elastic.sql.query.{ Predicate } import com.sksamuel.elastic4s.ElasticApi._ -import com.sksamuel.elastic4s.requests.common.FetchSourceContext import com.sksamuel.elastic4s.requests.searches.queries.{InnerHit, Query} -import scala.annotation.tailrec - case class ElasticBridge(filter: ElasticFilter) { def query( innerHitsNames: Set[String] = Set.empty, currentQuery: Option[ElasticBoolQuery] - )(implicit timestamp: Long): Query = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Query = { filter match { case boolQuery: ElasticBoolQuery => import boolQuery._ diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala index 32305e18..89e86e96 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala @@ -16,13 +16,15 @@ package app.softnetwork.elastic.sql.bridge +import app.softnetwork.elastic.sql.PainlessContextType import app.softnetwork.elastic.sql.query.Criteria import com.sksamuel.elastic4s.requests.searches.queries.Query case class ElasticCriteria(criteria: Criteria) { def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty)(implicit - timestamp: Long + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query ): Query = { val query = criteria.boolQuery.copy(group = group) query diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index d8de91e1..25c1903e 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -23,13 +23,13 @@ import app.softnetwork.elastic.sql.`type`.{ SQLTemporal, SQLVarchar } +import app.softnetwork.elastic.sql.config.ElasticSqlConfig import app.softnetwork.elastic.sql.function.aggregate.COUNT import app.softnetwork.elastic.sql.function.geo.{Distance, Meters} import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql.query._ import com.sksamuel.elastic4s.ElasticApi import com.sksamuel.elastic4s.ElasticApi._ -import com.sksamuel.elastic4s.requests.common.FetchSourceContext import com.sksamuel.elastic4s.requests.script.Script import com.sksamuel.elastic4s.requests.script.ScriptType.Source import com.sksamuel.elastic4s.requests.searches.aggs.{ @@ -51,17 +51,30 @@ import scala.language.implicitConversions package object bridge { - def now(script: Script)(implicit timestamp: Long): Script = { + lazy val sqlConfig: ElasticSqlConfig = ElasticSqlConfig() + + def now(script: Script)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Script = { if (!script.script.contains("params.__now__")) { return script } - script.param("__now__", timestamp) + contextType match { + case PainlessContextType.Query => script.param("__now__", timestamp) + case PainlessContextType.Transform => + script.param("__now__", sqlConfig.transformLastUpdatedColumnName) + case _ => script + } } implicit def requestToNestedFilterAggregation( request: SingleSearch, innerHitsName: String - )(implicit timestamp: Long): Option[FilterAggregation] = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Option[FilterAggregation] = { val having: Option[Query] = request.having.flatMap(_.criteria) match { case Some(f) => @@ -137,7 +150,10 @@ package object bridge { implicit def requestToFilterAggregation( request: SingleSearch - )(implicit timestamp: Long): Option[FilterAggregation] = + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Option[FilterAggregation] = request.having.flatMap(_.criteria) match { case Some(f) => val boolQuery = Option(ElasticBoolQuery(group = true)) @@ -155,7 +171,10 @@ package object bridge { implicit def requestToRootAggregations( request: SingleSearch, aggregations: Seq[ElasticAggregation] - )(implicit timestamp: Long): Seq[AbstractAggregation] = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Seq[AbstractAggregation] = { val notNestedAggregations = aggregations.filterNot(_.nested) val notNestedBuckets = request.bucketTree.filterNot(_.bucket.nested) @@ -207,7 +226,10 @@ package object bridge { implicit def requestToScopedAggregations( request: SingleSearch, aggregations: Seq[ElasticAggregation] - )(implicit timestamp: Long): Seq[NestedAggregation] = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Seq[NestedAggregation] = { // Group nested aggregations by their nested path val nestedAggregations: Map[String, Seq[ElasticAggregation]] = aggregations .filter(_.nested) @@ -413,7 +435,8 @@ package object bridge { } implicit def requestToElasticSearchRequest(request: SingleSearch)(implicit - timestamp: Long + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query ): ElasticSearchRequest = ElasticSearchRequest( request.sql, @@ -431,7 +454,10 @@ package object bridge { implicit def requestToSearchRequest( request: SingleSearch - )(implicit timestamp: Long): SearchRequest = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): SearchRequest = { import request._ val aggregations = request.aggregates.map( @@ -571,7 +597,10 @@ package object bridge { implicit def requestToMultiSearchRequest( request: MultiSearch - )(implicit timestamp: Long): MultiSearchRequest = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): MultiSearchRequest = { MultiSearchRequest( request.requests.map(implicitly[SearchRequest](_)) ) @@ -582,7 +611,10 @@ package object bridge { doubleOp: Double => A ): A = n.toEither.fold(longOp, doubleOp) - implicit def expressionToQuery(expression: GenericExpression)(implicit timestamp: Long): Query = { + implicit def expressionToQuery(expression: GenericExpression)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Query = { import expression._ if (isAggregation) return matchAllQuery() @@ -884,7 +916,10 @@ package object bridge { implicit def betweenToQuery( between: BetweenExpr - )(implicit timestamp: Long): Query = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Query = { import between._ // Geo distance special case identifier.functions.headOption match { @@ -1015,7 +1050,10 @@ package object bridge { implicit def sqlQueryToAggregations( query: SelectStatement - )(implicit timestamp: Long): Seq[ElasticAggregation] = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Seq[ElasticAggregation] = { import query._ statement .map { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala index 964a4967..3cf7e22b 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala @@ -24,7 +24,6 @@ import org.json4s.jackson import org.json4s.jackson.Serialization import org.slf4j.Logger -import java.io.Closeable import scala.language.{implicitConversions, postfixOps, reflectiveCalls} /** Created by smanciot on 28/06/2018. diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index a45c174a..c5888dfb 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -23,7 +23,7 @@ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.schema.Index -import app.softnetwork.elastic.sql.{query, schema} +import app.softnetwork.elastic.sql.{query, schema, PainlessContextType} import app.softnetwork.elastic.sql.query.{ DqlStatement, SQLAggregation, @@ -1322,10 +1322,13 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { )(implicit formats: Formats): ElasticResult[Seq[(U, Seq[I])]] = delegate.multisearchWithInnerHits[U, I](elasticQueries, innerField) - override private[client] implicit def sqlSearchRequestToJsonQuery( + override private[client] implicit def singleSearchToJsonQuery( sqlSearch: SingleSearch - )(implicit timestamp: Long): String = - delegate.sqlSearchRequestToJsonQuery(sqlSearch) + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): String = + delegate.singleSearchToJsonQuery(sqlSearch) override private[client] def executeSingleSearch( elasticQuery: ElasticQuery diff --git a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala index 680f4ee9..a895e6e8 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala @@ -21,6 +21,7 @@ import akka.stream.scaladsl.Source import app.softnetwork.elastic.client.bulk.BulkOptions import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.PainlessContextType import app.softnetwork.elastic.sql.parser.Parser import app.softnetwork.elastic.sql.query.{ Delete, @@ -1671,8 +1672,9 @@ trait IndicesApi extends ElasticClientHelpers { * @return * JSON string representation of the query */ - private[client] implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit - timestamp: Long + private[client] implicit def singleSearchToJsonQuery(sqlSearch: SingleSearch)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query ): String // ======================================================================== diff --git a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala index d74cf353..91151291 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala @@ -26,7 +26,7 @@ import app.softnetwork.elastic.client.bulk.{ FailedDocument, SuccessfulDocument } -import app.softnetwork.elastic.sql.query +import app.softnetwork.elastic.sql.{query, PainlessContextType} import app.softnetwork.elastic.sql.query.SQLAggregation import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ @@ -214,9 +214,12 @@ trait NopeClientApi extends ElasticClientApi { * @return * JSON string representation of the query */ - override private[client] implicit def sqlSearchRequestToJsonQuery( + override private[client] implicit def singleSearchToJsonQuery( sqlSearch: query.SingleSearch - )(implicit timestamp: Long): String = "{\"query\": {\"match_all\": {}}}" + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): String = "{\"query\": {\"match_all\": {}}}" override private[client] def executeUpdateSettings( index: String, diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala index 7977c791..19e2df5c 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala @@ -22,6 +22,7 @@ import app.softnetwork.elastic.client.result.{ ElasticResult, ElasticSuccess } +import app.softnetwork.elastic.sql.PainlessContextType import app.softnetwork.elastic.sql.macros.SQLQueryMacros import app.softnetwork.elastic.sql.query.{ DqlStatement, @@ -1035,8 +1036,9 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * @return * JSON string representation of the query */ - private[client] implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit - timestamp: Long + private[client] implicit def singleSearchToJsonQuery(sqlSearch: SingleSearch)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query ): String private def parseInnerHits[M: Manifest: ClassTag, I: Manifest: ClassTag]( diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index 792d3e0d..b7535b01 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.PainlessContext +import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType} import app.softnetwork.elastic.sql.`type`.SQLTemporal import app.softnetwork.elastic.sql.query.{ Asc, @@ -100,7 +100,10 @@ object ElasticAggregation { having: Option[Criteria], bucketsDirection: Map[String, SortOrder], allAggregations: Map[String, SQLAggregation] - )(implicit timestamp: Long): ElasticAggregation = { + )(implicit + timestamp: Long, + contextType: PainlessContextType + ): ElasticAggregation = { import sqlAgg._ val sourceField = identifier.path @@ -348,7 +351,10 @@ object ElasticAggregation { having: Option[Criteria], nested: Option[NestedElement], allElasticAggregations: Seq[ElasticAggregation] - )(implicit timestamp: Long): Seq[Aggregation] = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Seq[Aggregation] = { for (tree <- buckets) yield { val treeNodes = tree.sortBy(_.level).reverse.foldLeft(Seq.empty[NodeAggregation]) { (current, node) => diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala index ad5ad8ad..b5856a65 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala @@ -16,6 +16,7 @@ package app.softnetwork.elastic.sql.bridge +import app.softnetwork.elastic.sql.PainlessContextType import app.softnetwork.elastic.sql.operator.AND import app.softnetwork.elastic.sql.query.{ BetweenExpr, @@ -45,7 +46,10 @@ case class ElasticBridge(filter: ElasticFilter) { def query( innerHitsNames: Set[String] = Set.empty, currentQuery: Option[ElasticBoolQuery] - )(implicit timestamp: Long): Query = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Query = { filter match { case boolQuery: ElasticBoolQuery => import boolQuery._ diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala index f745889a..efb14be1 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala @@ -16,13 +16,15 @@ package app.softnetwork.elastic.sql.bridge +import app.softnetwork.elastic.sql.PainlessContextType import app.softnetwork.elastic.sql.query.Criteria import com.sksamuel.elastic4s.searches.queries.Query case class ElasticCriteria(criteria: Criteria) { def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty)(implicit - timestamp: Long + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query ): Query = { val query = criteria.boolQuery.copy(group = group) query diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 99e37c2c..09711ca8 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -23,6 +23,7 @@ import app.softnetwork.elastic.sql.`type`.{ SQLTemporal, SQLVarchar } +import app.softnetwork.elastic.sql.config.ElasticSqlConfig import app.softnetwork.elastic.sql.function.aggregate.COUNT import app.softnetwork.elastic.sql.function.geo.{Distance, Meters} import app.softnetwork.elastic.sql.operator._ @@ -47,17 +48,30 @@ import scala.language.implicitConversions package object bridge { - def now(script: Script)(implicit timestamp: Long): Script = { + lazy val sqlConfig: ElasticSqlConfig = ElasticSqlConfig() + + def now(script: Script)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Script = { if (!script.script.contains("params.__now__")) { return script } - script.param("__now__", timestamp) + contextType match { + case PainlessContextType.Query => script.param("__now__", timestamp) + case PainlessContextType.Transform => + script.param("__now__", sqlConfig.transformLastUpdatedColumnName) + case _ => script + } } implicit def requestToNestedFilterAggregation( request: SingleSearch, innerHitsName: String - )(implicit timestamp: Long): Option[FilterAggregation] = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Option[FilterAggregation] = { val having: Option[Query] = request.having.flatMap(_.criteria) match { case Some(f) => @@ -133,7 +147,10 @@ package object bridge { implicit def requestToFilterAggregation( request: SingleSearch - )(implicit timestamp: Long): Option[FilterAggregation] = + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Option[FilterAggregation] = request.having.flatMap(_.criteria) match { case Some(f) => val boolQuery = Option(ElasticBoolQuery(group = true)) @@ -151,7 +168,10 @@ package object bridge { implicit def requestToRootAggregations( request: SingleSearch, aggregations: Seq[ElasticAggregation] - )(implicit timestamp: Long): Seq[AbstractAggregation] = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Seq[AbstractAggregation] = { val notNestedAggregations = aggregations.filterNot(_.nested) val notNestedBuckets = request.bucketTree.filterNot(_.bucket.nested) @@ -201,7 +221,10 @@ package object bridge { implicit def requestToScopedAggregations( request: SingleSearch, aggregations: Seq[ElasticAggregation] - )(implicit timestamp: Long): Seq[NestedAggregation] = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Seq[NestedAggregation] = { // Group nested aggregations by their nested path val nestedAggregations: Map[String, Seq[ElasticAggregation]] = aggregations .filter(_.nested) @@ -407,7 +430,8 @@ package object bridge { } implicit def requestToElasticSearchRequest(request: SingleSearch)(implicit - timestamp: Long + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query ): ElasticSearchRequest = ElasticSearchRequest( request.sql, @@ -425,7 +449,10 @@ package object bridge { implicit def requestToSearchRequest( request: SingleSearch - )(implicit timestamp: Long): SearchRequest = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): SearchRequest = { import request._ val aggregations = request.aggregates.map( @@ -565,7 +592,10 @@ package object bridge { implicit def requestToMultiSearchRequest( request: MultiSearch - )(implicit timestamp: Long): MultiSearchRequest = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): MultiSearchRequest = { MultiSearchRequest( request.requests.map(implicitly[SearchRequest](_)) ) @@ -576,7 +606,10 @@ package object bridge { doubleOp: Double => A ): A = n.toEither.fold(longOp, doubleOp) - implicit def expressionToQuery(expression: GenericExpression)(implicit timestamp: Long): Query = { + implicit def expressionToQuery(expression: GenericExpression)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Query = { import expression._ if (isAggregation) return matchAllQuery() @@ -882,7 +915,10 @@ package object bridge { implicit def betweenToQuery( between: BetweenExpr - )(implicit timestamp: Long): Query = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Query = { import between._ // Geo distance special case identifier.functions.headOption match { @@ -1014,7 +1050,10 @@ package object bridge { @deprecated implicit def sqlQueryToAggregations( query: SelectStatement - )(implicit timestamp: Long): Seq[ElasticAggregation] = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Seq[ElasticAggregation] = { import query._ statement .map { diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala index 21809931..f69758e9 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala @@ -18,6 +18,7 @@ package app.softnetwork.elastic.client.jest import app.softnetwork.elastic.client.{ElasticQueries, ElasticQuery, SearchApi, SerializationApi} import app.softnetwork.elastic.client.result.ElasticResult +import app.softnetwork.elastic.sql.PainlessContextType import app.softnetwork.elastic.sql.bridge.ElasticSearchRequest import app.softnetwork.elastic.sql.query.SingleSearch import io.searchbox.core.MultiSearch @@ -29,9 +30,12 @@ import scala.language.implicitConversions trait JestSearchApi extends SearchApi with JestClientHelpers { _: JestClientCompanion with SerializationApi => - private[client] implicit def sqlSearchRequestToJsonQuery( + private[client] implicit def singleSearchToJsonQuery( sqlSearch: SingleSearch - )(implicit timestamp: Long): String = + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): String = implicitly[ElasticSearchRequest](sqlSearch).query import JestClientApi._ diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 66dc032f..a3a28bef 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -23,7 +23,7 @@ import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.sql.{ObjectValue, Value} +import app.softnetwork.elastic.sql.{ObjectValue, PainlessContextType, Value} import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.sql.schema.TableAlias @@ -942,8 +942,9 @@ trait RestHighLevelClientGetApi extends GetApi with RestHighLevelClientHelpers { trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHelpers { _: ElasticConversion with RestHighLevelClientCompanion with SerializationApi => - override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit - timestamp: Long + override implicit def singleSearchToJsonQuery(sqlSearch: SingleSearch)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query ): String = implicitly[ElasticSearchRequest](sqlSearch).query diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index c21aaf84..52c2db53 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -23,7 +23,7 @@ import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.sql.{ObjectValue, Value} +import app.softnetwork.elastic.sql.{ObjectValue, PainlessContextType, Value} import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.sql.schema.TableAlias @@ -930,10 +930,11 @@ trait RestHighLevelClientGetApi extends GetApi with RestHighLevelClientHelpers { trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHelpers { _: ElasticConversion with RestHighLevelClientCompanion with SerializationApi => - override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit - timestamp: Long + override implicit def singleSearchToJsonQuery(singleSearch: SingleSearch)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query ): String = - implicitly[ElasticSearchRequest](sqlSearch).query + implicitly[ElasticSearchRequest](singleSearch).query override private[client] def executeSingleSearch( elasticQuery: ElasticQuery diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 9005dee0..e8ddd8f9 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -31,6 +31,7 @@ import app.softnetwork.elastic.client.result.{ ElasticResult, ElasticSuccess } +import app.softnetwork.elastic.sql.PainlessContextType import app.softnetwork.elastic.sql.schema.TableAlias import app.softnetwork.elastic.sql.serialization._ import co.elastic.clients.elasticsearch._types.mapping.TypeMapping @@ -902,10 +903,11 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { _: JavaClientCompanion with SerializationApi => - override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit - timestamp: Long + override implicit def singleSearchToJsonQuery(singleSearch: SingleSearch)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query ): String = - implicitly[ElasticSearchRequest](sqlSearch).query + implicitly[ElasticSearchRequest](singleSearch).query override private[client] def executeSingleSearch( elasticQuery: ElasticQuery diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 563120b2..3a9a1eaa 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -31,6 +31,7 @@ import app.softnetwork.elastic.client.result.{ ElasticResult, ElasticSuccess } +import app.softnetwork.elastic.sql.PainlessContextType import app.softnetwork.elastic.sql.schema.TableAlias import app.softnetwork.elastic.sql.serialization._ import co.elastic.clients.elasticsearch._types.mapping.TypeMapping @@ -899,10 +900,11 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { _: JavaClientCompanion with SerializationApi => - override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit - timestamp: Long + override implicit def singleSearchToJsonQuery(singleSearch: SingleSearch)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query ): String = - implicitly[ElasticSearchRequest](sqlSearch).query + implicitly[ElasticSearchRequest](singleSearch).query override private[client] def executeSingleSearch( elasticQuery: ElasticQuery diff --git a/sql/src/main/resources/softnetwork-sql.conf b/sql/src/main/resources/softnetwork-sql.conf index 74fdf8e4..10c45b27 100644 --- a/sql/src/main/resources/softnetwork-sql.conf +++ b/sql/src/main/resources/softnetwork-sql.conf @@ -1,4 +1,5 @@ sql { composite-key-separator = "\\|\\|" # regex separator for composite keys in SQL ddl queries artificial-primary-key-column-name = "id" # name of the artificial primary key column added to tables without primary key(s) defined + transform-last-updated-column-name = "_last_updated" # name of the column used to track last updated timestamp for records } diff --git a/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala b/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala index 75c56386..92a5acb1 100644 --- a/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala +++ b/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala @@ -6,7 +6,8 @@ import configs.Configs case class ElasticSqlConfig( compositeKeySeparator: String, - artificialPrimaryKeyColumnName: String + artificialPrimaryKeyColumnName: String, + transformLastUpdatedColumnName: String ) object ElasticSqlConfig extends StrictLogging { diff --git a/sql/src/main/scala-2.13/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala b/sql/src/main/scala-2.13/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala index c777f29d..9fd99a49 100644 --- a/sql/src/main/scala-2.13/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala +++ b/sql/src/main/scala-2.13/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala @@ -22,7 +22,8 @@ import configs.ConfigReader case class ElasticSqlConfig( compositeKeySeparator: String, - artificialPrimaryKeyColumnName: String + artificialPrimaryKeyColumnName: String, + transformLastUpdatedColumnName: String ) object ElasticSqlConfig extends StrictLogging { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index ad54ce26..0cf32556 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -136,6 +136,7 @@ package object sql { case object PainlessContextType { case object Processor extends PainlessContextType case object Query extends PainlessContextType + case object Transform extends PainlessContextType } /** Context for painless scripts @@ -157,7 +158,8 @@ package object sql { lazy val timestamp: String = { context match { case PainlessContextType.Processor => CurrentFunction.processorTimestamp - case PainlessContextType.Query => CurrentFunction.queryTimestamp + case PainlessContextType.Query | PainlessContextType.Transform => + CurrentFunction.queryTimestamp } } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala index 023bb907..10c726c1 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala @@ -22,6 +22,7 @@ import akka.stream.scaladsl.{Flow, Source} import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result.ElasticResult import app.softnetwork.elastic.client.scroll._ +import app.softnetwork.elastic.sql.PainlessContextType import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.sql.schema.TableAlias import app.softnetwork.serialization._ @@ -290,9 +291,10 @@ trait MockElasticClientApi extends NopeClientApi { // ==================== SearchApi ==================== - override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: SingleSearch - )(implicit timestamp: Long): String = + override private[client] implicit def singleSearchToJsonQuery(singleSearch: SingleSearch)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): String = """{ | "query": { | "match_all": {} From bc70efd0d37ead2ce4b650038c9f497e3d19cf75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 12 Jan 2026 14:26:04 +0100 Subject: [PATCH 02/22] add standard join model --- .../elastic/sql/parser/FromParser.scala | 23 ++- .../softnetwork/elastic/sql/query/From.scala | 148 +++++++++++++++++- 2 files changed, 161 insertions(+), 10 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/FromParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/FromParser.scala index b18ea945..6d4fd964 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/FromParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/FromParser.scala @@ -16,6 +16,7 @@ package app.softnetwork.elastic.sql.parser +import app.softnetwork.elastic.sql.{Alias, Identifier} import app.softnetwork.elastic.sql.query.{ CrossJoin, From, @@ -26,6 +27,7 @@ import app.softnetwork.elastic.sql.query.{ LeftJoin, On, RightJoin, + StandardJoin, Table, Unnest } @@ -33,7 +35,7 @@ import app.softnetwork.elastic.sql.query.{ trait FromParser { self: Parser with WhereParser with LimitParser => - def unnest: PackratParser[Unnest] = + def unnest: PackratParser[Join] = Unnest.regex ~ start ~ identifier ~ end ~ alias.? ^^ { case _ ~ i ~ _ ~ a => Unnest(i, None, a) } @@ -52,8 +54,23 @@ trait FromParser { ) } - def join: PackratParser[Join] = opt(join_type) ~ Join.regex ~ unnest ~ opt(on) ^^ { - case jt ~ _ ~ t ~ o => t // Unnest cannot have a join type or an ON clause + def source: PackratParser[(Identifier, Option[Alias])] = identifier ~ alias.? ^^ { case i ~ a => + (i, a) + } + + def join: PackratParser[Join] = opt(join_type) ~ Join.regex ~ (unnest | source) ~ opt(on) ^^ { + case jt ~ _ ~ t ~ o => + t match { + case u: Unnest => + u // Unnest cannot have a join type or an ON clause + case (i: Identifier, a: Option[Alias]) => + StandardJoin( + source = i, + joinType = jt, + on = o, + alias = a + ) + } } def table: PackratParser[Table] = identifierRegex ~ alias.? ~ rep(join) ^^ { case i ~ a ~ js => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala index 3ad17781..87e4956d 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala @@ -16,12 +16,14 @@ package app.softnetwork.elastic.sql.query +import app.softnetwork.elastic.sql.operator.{AND, EQ} import app.softnetwork.elastic.sql.{ asString, Alias, Expr, Identifier, Source, + Token, TokenRegex, Updateable } @@ -44,9 +46,99 @@ case object CrossJoin extends Expr("CROSS") with JoinType case object On extends Expr("ON") with TokenRegex +case class JoinKey(table: String, tableAlias: String, field: String) extends Token { + def sql: String = s"$tableAlias.$field" + lazy val key: String = s"${table}_$field" +} + +object JoinKey { + def apply(identifier: Identifier): Option[JoinKey] = { + identifier.table match { + case Some(ta) if identifier.name.nonEmpty => + Some(JoinKey(ta, identifier.tableAlias.getOrElse(ta), identifier.name)) + case _ => None + } + } +} + +case class JoinKeyMatch(left: JoinKey, right: JoinKey) extends Token { + def sql: String = s"${left.sql} = ${right.sql}" + + lazy val joinKeys: Seq[JoinKey] = Seq(left, right) +} + +object JoinKeyMatch { + def apply(expression: Expression): Option[JoinKeyMatch] = { + if (expression.operator != EQ) return None + (expression.identifier, expression.maybeValue) match { + case (leftId: Identifier, Some(rightId: Identifier)) => + (JoinKey(leftId), JoinKey(rightId)) match { + case (Some(leftKey), Some(rightKey)) => + Some(JoinKeyMatch(leftKey, rightKey)) + case _ => None + } + case _ => None + } + } +} + case class On(criteria: Criteria) extends Updateable { override def sql: String = s" $On $criteria" def update(request: SingleSearch): On = this.copy(criteria = criteria.update(request)) + + private def extractJoinKeyMatches(criteria: Criteria): Seq[JoinKeyMatch] = { + criteria match { + case e: Expression if e.operator == EQ => JoinKeyMatch(e).toSeq + case p: Predicate if p.operator == AND => + extractJoinKeyMatches(p.leftCriteria) ++ extractJoinKeyMatches(p.rightCriteria) + case _ => Seq.empty + } + } + + /* Extracted join keys from the ON criteria */ + lazy val joinKeyMatches: Seq[JoinKeyMatch] = extractJoinKeyMatches(criteria) + + private def validateOnCriteria(criteria: Criteria): Either[String, Unit] = { + criteria match { + case e: Expression if e.operator == EQ => + if (e.identifier.name.isEmpty) Left(s"ON clause $this identifier cannot be empty") + else if (e.identifier.functions.nonEmpty) + Left(s"ON clause $this cannot use functions in equality expressions") + else if (e.maybeNot.isDefined) + Left(s"ON clause $this cannot use NOT operator in equality expressions") + else if (e.maybeValue.isEmpty) + Left(s"ON clause $this equality expressions must compare two identifiers") + else { + e.maybeValue.get match { + case id: Identifier => + if (id.name.isEmpty) + Left(s"ON clause $this identifier cannot be empty") + else if (id.functions.nonEmpty) + Left(s"ON clause $this cannot use functions in equality expressions") + else Right(()) + case _ => Left(s"ON clause $this equality expressions must compare two identifiers") + } + } + case p: Predicate => + p.operator match { + case AND => + for { + _ <- validateOnCriteria(p.leftCriteria) + _ <- validateOnCriteria(p.rightCriteria) + } yield () + case _ => Left(s"ON clause $this must use AND predicate operator") + } + case _ => Left(s"ON clause $this must use either equality operator or AND predicate") + } + } + + override def validate(): Either[String, Unit] = { + for { + _ <- super.validate() + _ <- criteria.validate() + _ <- validateOnCriteria(criteria) + } yield () + } } case object Join extends Expr("JOIN") with TokenRegex @@ -136,6 +228,36 @@ case class Unnest( } +case class StandardJoin( + source: Source, + joinType: Option[JoinType], // INNER JOIN by default + on: Option[On], + alias: Option[Alias] = None +) extends Join { + override def update(request: SingleSearch): StandardJoin = { + val updated = this.copy( + source = source.update(request), + on = on.map(_.update(request)) + ) + updated + } + + override def validate(): Either[String, Unit] = { + for { + _ <- joinType match { + case Some(InnerJoin | LeftJoin) => Right(()) + case None => Right(()) // by default INNER JOIN + case _ => Left(s"Standard JOIN $this requires an INNER (default) or LEFT JOIN type") + } + _ <- on match { + case Some(o) => o.validate() + case None => Left(s"Standard JOIN $this requires an ON clause") + } + _ <- super.validate() + } yield () + } +} + case class Table(name: String, tableAlias: Option[Alias] = None, joins: Seq[Join] = Nil) extends Source { override def sql: String = s"$name${asString(tableAlias)} ${joins.map(_.sql).mkString(" ")}".trim @@ -157,12 +279,7 @@ case class Table(name: String, tableAlias: Option[Alias] = None, joins: Seq[Join case class From(tables: Seq[Table]) extends Updateable { override def sql: String = s" $From ${tables.map(_.sql).mkString(",")}" - lazy val unnests: Seq[Unnest] = tables - .map(_.joins) - .collect { case j => - j.collect { case u: Unnest => u } - } - .flatten + lazy val unnests: Seq[Unnest] = joins.collect { case u: Unnest => u } lazy val tableAliases: Map[String, String] = tables .flatMap((table: Table) => @@ -171,7 +288,24 @@ case class From(tables: Seq[Table]) extends Updateable { case _ => Some(table.name -> table.name) } ) - .toMap ++ unnestAliases.map(unnest => unnest._2._1 -> unnest._1) + .toMap ++ unnestAliases.map(unnest => unnest._2._1 -> unnest._1) ++ joinAliases.map(join => + join._2._1 -> join._1 + ) + + lazy val aliasesToTable: Map[String, String] = tableAliases.map(_.swap) + + lazy val joins: Seq[Join] = tables.flatMap(_.joins) + + lazy val joinAliases: Map[String, (String, Option[On])] = joins.collect { case sj: StandardJoin => + ( + sj.alias + .map(_.alias) + .getOrElse( + sj.source.name + ), + (sj.source.name, sj.on) + ) + }.toMap lazy val unnestAliases: Map[String, (String, Option[Limit])] = unnests .map(u => // extract unnest info From 140f79405d6deb5afc42074ac3c7fe44ab99383c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 12 Jan 2026 14:36:58 +0100 Subject: [PATCH 03/22] add support for enrichment --- .../sql/function/aggregate/package.scala | 2 +- .../elastic/sql/function/package.scala | 55 ++-- .../app/softnetwork/elastic/sql/package.scala | 29 +- .../elastic/sql/parser/Parser.scala | 1 + .../softnetwork/elastic/sql/query/From.scala | 12 + .../elastic/sql/query/Select.scala | 7 +- .../elastic/sql/schema/package.scala | 264 ++++++++++++++++++ 7 files changed, 338 insertions(+), 32 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala index 16bf57b1..c9053de3 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala @@ -88,7 +88,7 @@ package object aggregate { aggregations.map(_.bucketPath).distinct.sortBy(_.length).reverse.headOption.getOrElse("") override def update(request: SingleSearch): BucketScriptAggregation = { - val identifiers = FunctionUtils.aggregateIdentifiers(identifier) + val identifiers = FunctionUtils.funIdentifiers(identifier) val params = identifiers.flatMap { case identifier: Identifier => val name = identifier.metricName.getOrElse(identifier.aliasOrName) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index 064e6b25..ef110b5e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -18,6 +18,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.function.aggregate.AggregateFunction +import app.softnetwork.elastic.sql.function.time.CurrentFunction import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression import app.softnetwork.elastic.sql.parser.Validator import app.softnetwork.elastic.sql.query.{NestedElement, SingleSearch} @@ -34,6 +35,11 @@ package object function { def expr: Token = _expr override def nullable: Boolean = expr.nullable def functionNestedElement: Option[NestedElement] = None + def dependencies: Seq[Identifier] = FunctionUtils.funIdentifiers(this).filterNot(_.name.isEmpty) + def usesCurrentTimeFunction: Boolean = this match { + case _: CurrentFunction => true + case _ => false + } } trait FunctionWithIdentifier extends Function { @@ -91,31 +97,29 @@ package object function { } } - def aggregateIdentifiers( + def funIdentifiers( fun: Function, - acc: Seq[FunctionChain] = Seq.empty - ): Seq[FunctionChain] = { + acc: Seq[Identifier] = Seq.empty + ): Seq[Identifier] = { fun match { - case fwi: FunctionWithIdentifier => aggregateIdentifiers(fwi.identifier, acc) - case fc: FunctionChain => - fc.functions.foldLeft(acc) { - case (innerAcc, _: AggregateFunction) => innerAcc :+ fc - case (innerAcc, i: FunctionWithIdentifier) => - aggregateIdentifiers(i.identifier, innerAcc) - case (innerAcc, fc: FunctionChain) => aggregateIdentifiers(fc, innerAcc) - case (innerAcc, b: BinaryFunction[_, _, _]) => aggregateIdentifiers(b, innerAcc) - case (innerAcc, _) => innerAcc + case id: Identifier => + id.functions.foldLeft(acc :+ id) { case (innerAcc, fun: Function) => + funIdentifiers(fun, innerAcc) } - case b: BinaryFunction[_, _, _] => - val leftAcc = b.left match { - case f: Function => aggregateIdentifiers(f, acc) - case _ => acc - } - b.right match { - case f: Function => aggregateIdentifiers(f, leftAcc) - case _ => leftAcc + case fn: FunctionN[_, _] => + fn.args + .collect { case f: Function => + f + } + .foldLeft(acc) { (innerAcc, f) => + funIdentifiers(f, innerAcc) + } + case fc: FunctionChain => + fc.functions.foldLeft(acc) { case (innerAcc, fun: Function) => + funIdentifiers(fun, innerAcc) } - case _ => acc + case fwi: FunctionWithIdentifier => funIdentifiers(fwi.identifier, acc :+ fwi.identifier) + case _ => acc } } } @@ -205,6 +209,10 @@ package object function { override def functionNestedElement: Option[NestedElement] = functions.flatMap(_.functionNestedElement).headOption + + override def usesCurrentTimeFunction: Boolean = { + functions.exists { _.usesCurrentTimeFunction } + } } trait FunctionN[In <: SQLType, Out <: SQLType] extends Function with PainlessScript { @@ -350,6 +358,11 @@ package object function { s"${fun.map(_.painless(context)).getOrElse("")}(${callArgs.mkString(argsSeparator)})" else fun.map(_.painless(context)).getOrElse("") + + override def usesCurrentTimeFunction: Boolean = this.args.exists { + case f: Function => f.usesCurrentTimeFunction + case _ => false + } } trait BinaryFunction[In1 <: SQLType, In2 <: SQLType, Out <: SQLType] extends FunctionN[In2, Out] { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 0cf32556..ccd399ca 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -805,8 +805,9 @@ package object sql { // Tronquer si nécessaire if (normalized.length > MaxAliasLength) { val digest = MessageDigest.getInstance("MD5").digest(normalized.getBytes("UTF-8")) - val hash = digest.map("%02x".format(_)).mkString.take(8) // suffix court - normalized.take(MaxAliasLength - hash.length - 1) + "_" + hash + val hash = digest.map("%02x".format(_)).mkString.take(MaxAliasLength) + //normalized.take(MaxAliasLength - hash.length - 1) + "_" + hash + hash } else { normalized } @@ -1076,6 +1077,7 @@ package object sql { def isWindowing: Boolean = windows.exists(_.partitionBy.nonEmpty) + def painlessScriptRequired: Boolean = functions.nonEmpty && !hasAggregation && bucket.isEmpty } object Identifier { @@ -1099,7 +1101,8 @@ package object sql { nestedElement: Option[NestedElement] = None, bucketPath: String = "", col: Option[Column] = None, - table: Option[String] = None + table: Option[String] = None, + override val dependencies: Seq[Identifier] = Seq.empty ) extends Identifier { def withFunctions(functions: List[Function]): Identifier = this.copy(functions = functions) @@ -1129,6 +1132,11 @@ package object sql { val parts: Seq[String] = name.split("\\.").toSeq val tableAlias = parts.head val table = request.tableAliases.find(t => t._2 == tableAlias).map(_._2) + val dependencies = functions + .foldLeft(Seq.empty[Identifier]) { case (acc, fun) => + acc ++ FunctionUtils.funIdentifiers(fun) + } + .filterNot(_.name.isEmpty) if (table.nonEmpty) { request.unnestAliases.find(_._1 == tableAlias) match { case Some(tuple) if !nested => @@ -1149,7 +1157,8 @@ package object sql { nestedElement = nestedElement, bucketPath = bucketPath, col = request.schema.flatMap(schema => schema.find(colName)), - table = table + table = table, + dependencies = dependencies.map(_.update(request)) ) .withFunctions(this.updateFunctions(request)) case Some(tuple) if nested => @@ -1163,7 +1172,8 @@ package object sql { bucket = request.bucketNames.get(identifierName).orElse(bucket), bucketPath = bucketPath, col = request.schema.flatMap(schema => schema.find(colName)), - table = table + table = table, + dependencies = dependencies.map(_.update(request)) ) .withFunctions(this.updateFunctions(request)) case None if nested => @@ -1174,7 +1184,8 @@ package object sql { bucket = request.bucketNames.get(identifierName).orElse(bucket), bucketPath = bucketPath, col = request.schema.flatMap(schema => schema.find(name)), - table = table + table = table, + dependencies = dependencies.map(_.update(request)) ) .withFunctions(this.updateFunctions(request)) case _ => @@ -1186,7 +1197,8 @@ package object sql { bucket = request.bucketNames.get(identifierName).orElse(bucket), bucketPath = bucketPath, col = request.schema.flatMap(schema => schema.find(colName)), - table = table + table = table, + dependencies = dependencies.map(_.update(request)) ) } } else { @@ -1195,7 +1207,8 @@ package object sql { fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias), bucket = request.bucketNames.get(identifierName).orElse(bucket), bucketPath = bucketPath, - col = request.schema.flatMap(schema => schema.find(name)) + col = request.schema.flatMap(schema => schema.find(name)), + dependencies = dependencies.map(_.update(request)) ) .withFunctions(this.updateFunctions(request)) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 81ddacf5..e01f33a2 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -133,6 +133,7 @@ object Parser case "rename" => IngestProcessorType.Rename case "remove" => IngestProcessorType.Remove case "date_index_name" => IngestProcessorType.DateIndexName + case "enrich" => IngestProcessorType.Enrich case other => IngestProcessorType(other) } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala index 87e4956d..d9beac93 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala @@ -275,6 +275,13 @@ case class Table(name: String, tableAlias: Option[Alias] = None, joins: Seq[Join case errors => Left(errors.map { case Left(err) => err }.mkString("\n")) } } yield () + + lazy val joinedTables: Seq[String] = joins.collect { case sj: StandardJoin => + sj.source.name + } + + lazy val enrichmentRequired: Boolean = joinedTables.nonEmpty + } case class From(tables: Seq[Table]) extends Updateable { @@ -312,6 +319,7 @@ case class From(tables: Seq[Table]) extends Updateable { (u.alias.map(_.alias).getOrElse(u.name), (u.name, u.limit)) ) .toMap + def update(request: SingleSearch): From = this.copy(tables = tables.map(_.update(request))) @@ -331,6 +339,10 @@ case class From(tables: Seq[Table]) extends Updateable { } lazy val mainTable: Table = tables.head + + lazy val joinedTables: Seq[String] = tables.flatMap(_.joinedTables) + + lazy val enrichmentRequired: Boolean = joinedTables.nonEmpty } case class NestedElement( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala index 45d47194..8fbf774f 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala @@ -47,9 +47,11 @@ case class Field( with DateMathScript { def tableAlias: Option[String] = identifier.tableAlias def table: Option[String] = identifier.table - def isScriptField: Boolean = - functions.nonEmpty && !hasAggregation && identifier.bucket.isEmpty + def isScriptField: Boolean = identifier.painlessScriptRequired override def sql: String = s"$identifier${asString(fieldAlias)}" + + override def dependencies: Seq[Identifier] = identifier.dependencies + lazy val sourceField: String = { if (identifier.nested) { tableAlias @@ -126,6 +128,7 @@ case class Select( lazy val fieldAliases: Map[String, String] = fields.flatMap { field => field.fieldAlias.map(a => field.identifier.identifierName -> a.alias) }.toMap + lazy val aliasesToMap: Map[String, String] = fieldAliases.map(_.swap) def update(request: SingleSearch): Select = this.copy(fields = fields.map(_.update(request)), except = except.map(_.update(request))) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 5b2ebf93..ed17f243 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -688,6 +688,270 @@ package object schema { } } + sealed trait EnrichPolicyType { + def name: String + override def toString: String = name + } + + object EnrichPolicyType { + case object Match extends EnrichPolicyType { + val name: String = "MATCH" + } + case object GeoMatch extends EnrichPolicyType { + val name: String = "GEO_MATCH" + } + case object Range extends EnrichPolicyType { + val name: String = "RANGE" + } + } + + case class EnrichPolicy( + name: String, + policyType: EnrichPolicyType = EnrichPolicyType.Match, + indices: Seq[String], + matchField: String, + enrichFields: List[String], + criteria: Option[Criteria] = None + ) extends DdlToken { + def sql: String = + s"CREATE OR REPLACE ENRICH POLICY $name TYPE $policyType WITH SOURCE INDICES ${indices + .mkString(",")} MATCH FIELD $matchField ENRICH FIELDS (${enrichFields.mkString(", ")})${Where(criteria)}" + } + + case class Pivot( + fields: Seq[Column], // simple columns to add to the destination index (within aggregations) + aggregations: Seq[SQLAggregation], // aggregations to apply + groupBy: Option[GroupBy] // group by clause + ) extends DdlToken { + def sql: String = { + val groupByStr = if (groupBy.nonEmpty) { + s"GROUP BY (${groupBy.mkString(", ")})" + } else { + "" + } + val aggregationsStr = if (aggregations.nonEmpty) { + s"AGGREGATIONS (${aggregations.mkString(", ")})" + } else { + "" + } + s"$groupByStr $aggregationsStr".trim + } + } + + sealed trait TransformTimeUnit extends DdlToken { + def name: String + } + + object TransformTimeUnit { + case object Milliseconds extends TransformTimeUnit { + val name: String = "MILLISECONDS" + override def sql: String = "ms" + } + + case object Seconds extends TransformTimeUnit { + val name: String = "SECONDS" + override def sql: String = "s" + } + + case object Minutes extends TransformTimeUnit { + val name: String = "MINUTES" + override def sql: String = "m" + } + + case object Hours extends TransformTimeUnit { + val name: String = "HOURS" + override def sql: String = "h" + } + + case object Days extends TransformTimeUnit { + val name: String = "DAYS" + override def sql: String = "d" + } + + case object Weeks extends TransformTimeUnit { + val name: String = "WEEKS" + override def sql: String = "w" + } + + def apply(name: String): TransformTimeUnit = name.toUpperCase() match { + case "MILLISECONDS" => Milliseconds + case "SECONDS" => Seconds + case "MINUTES" => Minutes + case "HOURS" => Hours + case "DAYS" => Days + case "WEEKS" => Weeks + case other => throw new IllegalArgumentException(s"Invalid delay unit: $other") + } + } + + sealed trait TransformTimeInterval extends DdlToken { + def interval: Int + def timeUnit: TransformTimeUnit + def toSeconds: Int = timeUnit match { + case TransformTimeUnit.Milliseconds => interval / 1000 + case TransformTimeUnit.Seconds => interval + case TransformTimeUnit.Minutes => interval * 60 + case TransformTimeUnit.Hours => interval * 3600 + case TransformTimeUnit.Days => interval * 86400 + case TransformTimeUnit.Weeks => interval * 604800 + } + def toTransformFormat: String = s"$interval$timeUnit" + + } + + object TransformTimeInterval { + + /** Creates a time interval from seconds */ + def fromSeconds(seconds: Int): (TransformTimeUnit, Int) = { + if (seconds >= 86400 && seconds % 86400 == 0) { + (TransformTimeUnit.Days, seconds / 86400) + } else if (seconds >= 3600 && seconds % 3600 == 0) { + (TransformTimeUnit.Hours, seconds / 3600) + } else if (seconds >= 60 && seconds % 60 == 0) { + (TransformTimeUnit.Minutes, seconds / 60) + } else { + (TransformTimeUnit.Seconds, seconds) + } + } + } + + case class Delay( + timeUnit: TransformTimeUnit, + interval: Int + ) extends TransformTimeInterval { + def sql: String = s"WITH DELAY $interval $timeUnit" + } + + object Delay { + val Default: Delay = Delay(TransformTimeUnit.Minutes, 1) + + def fromSeconds(seconds: Int): Delay = { + val timeInterval = TransformTimeInterval.fromSeconds(seconds) + Delay(timeInterval._1, timeInterval._2) + } + + /** Calculates optimal delay based on frequency and number of stages + * + * Formula: delay = frequency / (nb_stages * buffer_factor) + * + * This ensures the complete chain can refresh within the specified frequency. The buffer + * factor adds safety margin for processing time. + * + * @param frequency + * Desired refresh frequency + * @param nbStages + * Total number of stages (changelog + enrichment + aggregate) + * @param bufferFactor + * Safety factor (default 1.5) + * @return + * Optimal delay for each Transform, or error if constraints cannot be met + */ + def calculateOptimal( + frequency: Frequency, + nbStages: Int, + bufferFactor: Double = 1.5 + ): Either[String, Delay] = { + if (nbStages <= 0) { + return Left("Number of stages must be positive") + } + + val frequencySeconds = frequency.toSeconds + val optimalDelaySeconds = (frequencySeconds / (nbStages * bufferFactor)).toInt + + // Validate constraints + if (optimalDelaySeconds < 10) { + Left( + s"Calculated delay ($optimalDelaySeconds seconds) is too small. " + + s"Consider increasing frequency or reducing number of stages. " + + s"Minimum required frequency: ${nbStages * bufferFactor * 10} seconds" + ) + } else if (optimalDelaySeconds > frequencySeconds / 2) { + Left( + s"Calculated delay ($optimalDelaySeconds seconds) is too large. " + + s"Each stage needs at least delay × 2 = frequency." + ) + } else { + Right(Delay.fromSeconds(optimalDelaySeconds)) + } + } + + /** Validates that a chain of transforms can refresh within the given frequency + * + * Requirements: + * - Total latency (delay × nbStages) must be less than frequency + * - Each transform runs every (delay × 2), so: delay × 2 × nbStages ≤ frequency + * + * @param delay + * Delay for each transform + * @param frequency + * Target refresh frequency + * @param nbStages + * Number of stages in the chain + * @return + * Success or error message + */ + def validate( + delay: Delay, + frequency: Frequency, + nbStages: Int + ): Either[String, Unit] = { + val totalLatency = delay.toSeconds * nbStages + val frequencySeconds = frequency.toSeconds + val requiredFrequency = delay.toSeconds * 2 * nbStages + + if (totalLatency > frequencySeconds) { + Left( + s"Total latency ($totalLatency seconds) exceeds frequency ($frequencySeconds seconds). " + + s"Minimum required frequency: $requiredFrequency seconds" + ) + } else if (requiredFrequency > frequencySeconds) { + Left( + s"Frequency ($frequencySeconds seconds) is too low for $nbStages stages with delay ${delay.toSeconds} seconds. " + + s"Minimum required frequency: $requiredFrequency seconds" + ) + } else { + Right(()) + } + } + } + + case class Frequency( + timeUnit: TransformTimeUnit, + interval: Int + ) extends TransformTimeInterval { + def sql: String = s"REFRESH EVERY $interval $timeUnit" + } + + case object Frequency { + val Default: Frequency = apply(Delay.Default) + def apply(delay: Delay): Frequency = Frequency(delay.timeUnit, delay.interval * 2) + def fromSeconds(seconds: Int): Frequency = { + val timeInterval = TransformTimeInterval.fromSeconds(seconds) + Frequency(timeInterval._1, timeInterval._2) + } + } + + /** Definition of a column within a table + * + * @param name + * the column name + * @param dataType + * the column SQL type + * @param script + * optional script processor associated to this column + * @param multiFields + * optional multi fields associated to this column + * @param defaultValue + * optional default value for this column + * @param notNull + * whether this column is not null + * @param comment + * optional comment for this column + * @param options + * optional options for this column (search analyzer, ...) + * @param struct + * optional parent struct column + */ case class Column( name: String, dataType: SQLType, From 3faf30b8a1b3ee06cab0f1f660fcef31dfa8a0f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 12 Jan 2026 15:13:40 +0100 Subject: [PATCH 04/22] to fix compilation bugs with scala 2.12 --- .../main/scala/app/softnetwork/elastic/client/BulkApi.scala | 2 ++ .../app/softnetwork/elastic/client/ElasticClientHelpers.scala | 2 +- .../main/scala/app/softnetwork/elastic/schema/package.scala | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/BulkApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/BulkApi.scala index c396d50a..6973c95e 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/BulkApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/BulkApi.scala @@ -31,6 +31,8 @@ import java.time.LocalDate import java.time.format.DateTimeFormatter import scala.concurrent.{ExecutionContext, Future} +import scala.language.implicitConversions + /** Bulk API for Elasticsearch clients. */ trait BulkApi extends BulkTypes with ElasticClientHelpers { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala index 23f82066..3747ecfd 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala @@ -239,7 +239,7 @@ trait ElasticClientHelpers { val pattern = "^[a-zA-Z0-9._\\-@]+$".r - if (!pattern.matches(trimmed)) { + if (!pattern.pattern.matcher(trimmed).matches()) { return Some( ElasticError( message = diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index 897b8392..304bda0c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -454,6 +454,8 @@ package object schema { } + val aliases = esAliases.aliases.map(entry => entry._1 -> implicitly[Value[_]](entry._2.node)) + // 4. Final construction of the Table Table( name = name, @@ -463,7 +465,7 @@ package object schema { mappings = esMappings.options, settings = esSettings.options, processors = processors.toSeq, - aliases = esAliases.aliases.map(entry => entry._1 -> entry._2.node) + aliases = aliases ).update() } } From 2b7f66b8bcc6049bc32ae703e401c5b992ca070e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 12 Jan 2026 16:35:21 +0100 Subject: [PATCH 05/22] add transform model --- .../elastic/sql/schema/package.scala | 291 ++++++++++++++++-- 1 file changed, 272 insertions(+), 19 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index ed17f243..d597a3f1 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -18,6 +18,14 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.config.ElasticSqlConfig +import app.softnetwork.elastic.sql.function.aggregate.{ + AggregateFunction, + AvgAgg, + CountAgg, + MaxAgg, + MinAgg, + SumAgg +} import app.softnetwork.elastic.sql.query._ import app.softnetwork.elastic.sql.serialization._ import app.softnetwork.elastic.sql.time.TimeUnit @@ -718,25 +726,7 @@ package object schema { .mkString(",")} MATCH FIELD $matchField ENRICH FIELDS (${enrichFields.mkString(", ")})${Where(criteria)}" } - case class Pivot( - fields: Seq[Column], // simple columns to add to the destination index (within aggregations) - aggregations: Seq[SQLAggregation], // aggregations to apply - groupBy: Option[GroupBy] // group by clause - ) extends DdlToken { - def sql: String = { - val groupByStr = if (groupBy.nonEmpty) { - s"GROUP BY (${groupBy.mkString(", ")})" - } else { - "" - } - val aggregationsStr = if (aggregations.nonEmpty) { - s"AGGREGATIONS (${aggregations.mkString(", ")})" - } else { - "" - } - s"$groupByStr $aggregationsStr".trim - } - } + // ==================== Transform ==================== sealed trait TransformTimeUnit extends DdlToken { def name: String @@ -931,6 +921,269 @@ package object schema { } } + case class TransformSource( + index: Seq[String], + query: Option[Criteria] + ) extends DdlToken { + override def sql: String = { + val queryStr = query.map(q => s" WHERE ${q.sql}").getOrElse("") + s"INDEX (${index.mkString(", ")})$queryStr" + } + + /** Converts to JSON for Elasticsearch + */ + def toJson(implicit criteriaToMap: Criteria => Map[String, Any]): Map[String, Any] = { + Map( + "index" -> index + ) ++ query.map(q => "query" -> implicitly[Map[String, Any]](q)) + } + } + + case class TransformDest( + index: String, + pipeline: Option[String] = None + ) extends DdlToken { + override def sql: String = { + val pipelineStr = pipeline.map(p => s" PIPELINE $p").getOrElse("") + s"INDEX $index$pipelineStr" + } + def toJson: Map[String, Any] = Map( + "index" -> index + ) ++ pipeline.map(p => "pipeline" -> p) + } + + /** Configuration for bucket selector (HAVING clause) + */ + case class TransformBucketSelectorConfig( + name: String = "having_filter", + bucketsPath: Map[String, String], + having: Criteria + ) { + def toJson(implicit criteriaToMap: Criteria => Map[String, Any]): Map[String, Any] = { + + Map( + "bucket_selector" -> Map( + "buckets_path" -> bucketsPath, + "script" -> implicitly[Map[String, Any]](having) + ) + ) + } + } + + case class TransformPivot( + groupBy: Map[String, TransformGroupBy], + aggregations: Map[String, TransformAggregation], + bucketSelector: Option[TransformBucketSelectorConfig] = None + ) extends DdlToken { + override def sql: String = { + val groupByStr = groupBy + .map { case (name, gb) => + s"$name BY ${gb.sql}" + } + .mkString(", ") + + val aggStr = aggregations + .map { case (name, agg) => + s"$name AS ${agg.sql}" + } + .mkString(", ") + + val havingStr = bucketSelector.map(bs => s" HAVING ${bs.having}").getOrElse("") + + s"PIVOT ($groupByStr) AGGREGATE ($aggStr)$havingStr" + } + + /** Converts to JSON for Elasticsearch + */ + def toJson(implicit criteriaToMap: Criteria => Map[String, Any]): Map[String, Any] = { + Map( + "group_by" -> groupBy.map { case (name, gb) => + name -> gb.toJson + }, + "aggregations" -> (aggregations.map { case (name, agg) => + name -> agg.toJson + } ++ bucketSelector.map(bs => bs.name -> bs.toJson)) + ) + } + } + + sealed trait TransformGroupBy extends DdlToken { + def toJson: Map[String, Any] + } + + case class TermsGroupBy(field: String) extends TransformGroupBy { + override def sql: String = s"TERMS($field)" + override def toJson: Map[String, Any] = Map("terms" -> Map("field" -> field)) + } + + sealed trait TransformAggregation extends DdlToken { + def toJson: Map[String, Any] + } + + case class MaxTransformAggregation(field: String) extends TransformAggregation { + override def sql: String = s"MAX($field)" + + override def toJson: Map[String, Any] = Map( + "max" -> Map("field" -> field) + ) + } + + case class MinTransformAggregation(field: String) extends TransformAggregation { + override def sql: String = s"MIN($field)" + + override def toJson: Map[String, Any] = Map( + "min" -> Map("field" -> field) + ) + } + + case class SumTransformAggregation(field: String) extends TransformAggregation { + override def sql: String = s"SUM($field)" + + override def toJson: Map[String, Any] = Map( + "sum" -> Map("field" -> field) + ) + } + + case class AvgTransformAggregation(field: String) extends TransformAggregation { + override def sql: String = s"AVG($field)" + + override def toJson: Map[String, Any] = Map( + "avg" -> Map("field" -> field) + ) + } + + case class CountTransformAggregation(field: String) extends TransformAggregation { + override def sql: String = if (field == "_id") "COUNT(*)" else s"COUNT($field)" + + override def toJson: Map[String, Any] = Map( + "value_count" -> Map("field" -> field) + ) + } + + case class CardinalityTransformAggregation(field: String) extends TransformAggregation { + override def sql: String = s"COUNT(DISTINCT $field)" + + override def toJson: Map[String, Any] = Map( + "cardinality" -> Map("field" -> field) + ) + } + + /** Extension methods for AggregateFunction + */ + implicit class AggregateConversion(agg: AggregateFunction) { + + /** Converts SQL aggregate function to Elasticsearch aggregation + */ + def toTransformAggregation: Option[TransformAggregation] = agg match { + case ma: MaxAgg => + Some(MaxTransformAggregation(ma.identifier.name)) + + case ma: MinAgg => + Some(MinTransformAggregation(ma.identifier.name)) + + case sa: SumAgg => + Some(SumTransformAggregation(sa.identifier.name)) + + case aa: AvgAgg => + Some(AvgTransformAggregation(aa.identifier.name)) + + case ca: CountAgg => + val field = ca.identifier.name + Some( + if (field == "*" || field.isEmpty) + if (ca.isCardinality) CardinalityTransformAggregation("_id") + else CountTransformAggregation("_id") + else if (ca.isCardinality) CardinalityTransformAggregation(field) + else CountTransformAggregation(field) + ) + + case _ => + // For other aggregate functions, default to none + None + } + } + + case class TransformSync(time: TransformTimeSync) extends DdlToken { + override def sql: String = s"SYNC ${time.sql}" + def toJson: Map[String, Any] = { + Map( + "time" -> time.toJson + ) + } + } + + case class TransformTimeSync(field: String, delay: Delay) extends DdlToken { + override def sql: String = s"TIME FIELD $field ${delay.sql}" + def toJson: Map[String, Any] = { + Map( + "field" -> field, + "delay" -> delay.toTransformFormat + ) + } + } + + /** Transform configuration models with built-in JSON serialization + */ + case class TransformConfig( + id: String, + source: TransformSource, + dest: TransformDest, + pivot: Option[TransformPivot] = None, + sync: Option[TransformSync] = None, + delay: Delay, + frequency: Frequency + ) extends DdlToken { + + /** Delay in seconds for display */ + lazy val delaySeconds: Int = delay.toSeconds + + /** Frequency in seconds for display */ + lazy val frequencySeconds: Int = frequency.toSeconds + + /** Converts to Elasticsearch JSON format + */ + def toJson(implicit criteriaToMap: Criteria => Map[String, Any]): Map[String, Any] = { + Map( + "source" -> source.toJson, + "dest" -> dest.toJson, + "frequency" -> frequency.toTransformFormat, + "sync" -> sync.map(_.toJson) + ) ++ pivot.map(p => "pivot" -> p.toJson) + } + + /** SQL DDL representation + */ + override def sql: String = { + val sb = new StringBuilder() + + sb.append(s"CREATE TRANSFORM $id\n") + sb.append(s" SOURCE ${source.sql}\n") + sb.append(s" DEST ${dest.sql}\n") + + pivot.foreach { p => + sb.append(s" ${p.sql}\n") + } + + sync.foreach { s => + sb.append(s" ${s.sql}\n") + } + + sb.append(s" ${frequency.sql}\n") + sb.append(s" ${delay.sql}") + + sb.toString() + } + + /** Human-readable summary + */ + def summary: String = { + s"Transform $id: ${source.index.mkString(", ")} → ${dest.index} " + + s"(every ${frequencySeconds}s, delay ${delaySeconds}s)" + } + } + + // ==================== Schema ==================== + /** Definition of a column within a table * * @param name From fdcf3dde0bd4f50028c5e2f3a06c5672a9995896 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 13 Jan 2026 07:41:57 +0100 Subject: [PATCH 06/22] replace toJson: Map by node: JsonNode for transform model --- .../softnetwork/elastic/schema/package.scala | 2 +- .../sql/function/aggregate/package.scala | 2 + .../elastic/sql/schema/package.scala | 184 +++++++++++------- 3 files changed, 116 insertions(+), 72 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index 304bda0c..bb27f971 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -433,7 +433,7 @@ package object schema { enrichedCols.update(col, c.copy(script = Some(p))) } - case p: SetProcessor => + case p: SetProcessor if p.isDefault => val col = p.column enrichedCols.get(col).foreach { c => enrichedCols.update(col, c.copy(defaultValue = Some(p.value))) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala index c9053de3..a74d02cb 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala @@ -226,6 +226,8 @@ package object aggregate { partitionBy: Seq[Identifier] = Seq.empty, fields: Seq[Field] = Seq.empty ) extends WindowFunction { + def isCardinality: Boolean = identifier.distinct + override def limit: Option[Limit] = None override def orderBy: Option[OrderBy] = None diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index d597a3f1..96a40a36 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -437,7 +437,14 @@ package object schema { ) extends IngestProcessor { def processorType: IngestProcessorType = IngestProcessorType.Set + def isDefault: Boolean = copyFrom.isEmpty && value != Null && (doIf match { + case Some(i) if i.contains(s"ctx.$column == null") => true + case _ => false + }) + override def sql: String = { + if (isDefault) + return s"$column SET DEFAULT ${value.sql}" val base = copyFrom match { case Some(source) => s"$column COPY FROM $source" case None => s"$column SET VALUE ${value.sql}" @@ -586,6 +593,7 @@ package object schema { val processorsNode = mapper.createArrayNode() processors.foreach { processor => processorsNode.add(processor.node) + () } node.put("description", sql) node.set("processors", processorsNode) @@ -932,10 +940,16 @@ package object schema { /** Converts to JSON for Elasticsearch */ - def toJson(implicit criteriaToMap: Criteria => Map[String, Any]): Map[String, Any] = { - Map( - "index" -> index - ) ++ query.map(q => "query" -> implicitly[Map[String, Any]](q)) + def node(implicit criteriaToNode: Criteria => JsonNode): JsonNode = { + val node = mapper.createObjectNode() + val indicesNode = mapper.createArrayNode() + index.foreach(indicesNode.add) + node.set("index", indicesNode) + query.foreach { q => + node.set("query", implicitly[JsonNode](q)) + () + } + node } } @@ -947,9 +961,12 @@ package object schema { val pipelineStr = pipeline.map(p => s" PIPELINE $p").getOrElse("") s"INDEX $index$pipelineStr" } - def toJson: Map[String, Any] = Map( - "index" -> index - ) ++ pipeline.map(p => "pipeline" -> p) + def node: JsonNode = { + val node = mapper.createObjectNode() + node.put("index", index) + pipeline.foreach(p => node.put("pipeline", p)) + node + } } /** Configuration for bucket selector (HAVING clause) @@ -959,14 +976,18 @@ package object schema { bucketsPath: Map[String, String], having: Criteria ) { - def toJson(implicit criteriaToMap: Criteria => Map[String, Any]): Map[String, Any] = { - - Map( - "bucket_selector" -> Map( - "buckets_path" -> bucketsPath, - "script" -> implicitly[Map[String, Any]](having) - ) - ) + def node(implicit criteriaToNode: Criteria => JsonNode): JsonNode = { + val node = mapper.createObjectNode() + val bucketSelectorNode = mapper.createObjectNode() + val bucketsPathNode = mapper.createObjectNode() + bucketsPath.foreach { case (k, v) => + bucketsPathNode.put(k, v) + () + } + bucketSelectorNode.set("buckets_path", bucketsPathNode) + bucketSelectorNode.set("script", implicitly[JsonNode](having)) + node.set("bucket_selector", bucketSelectorNode) + node } } @@ -995,77 +1016,89 @@ package object schema { /** Converts to JSON for Elasticsearch */ - def toJson(implicit criteriaToMap: Criteria => Map[String, Any]): Map[String, Any] = { - Map( - "group_by" -> groupBy.map { case (name, gb) => - name -> gb.toJson - }, - "aggregations" -> (aggregations.map { case (name, agg) => - name -> agg.toJson - } ++ bucketSelector.map(bs => bs.name -> bs.toJson)) - ) + def node(implicit criteriaToNode: Criteria => JsonNode): JsonNode = { + val node = mapper.createObjectNode() + + val groupByNode = mapper.createObjectNode() + groupBy.foreach { case (name, gb) => + groupByNode.set(name, gb.node) + () + } + node.set("group_by", groupByNode) + + val aggsNode = mapper.createObjectNode() + aggregations.foreach { case (name, agg) => + aggsNode.set(name, agg.node) + () + } + bucketSelector.foreach { bs => + aggsNode.set(bs.name, bs.node) + () + } + node.set("aggregations", aggsNode) + + node } } sealed trait TransformGroupBy extends DdlToken { - def toJson: Map[String, Any] + def node: JsonNode } case class TermsGroupBy(field: String) extends TransformGroupBy { override def sql: String = s"TERMS($field)" - override def toJson: Map[String, Any] = Map("terms" -> Map("field" -> field)) + + override def node: JsonNode = { + val node = mapper.createObjectNode() + val termsNode = mapper.createObjectNode() + termsNode.put("field", field) + node.set("terms", termsNode) + node + } } sealed trait TransformAggregation extends DdlToken { - def toJson: Map[String, Any] + def name: String + + def field: String + + override def sql: String = s"${name.toUpperCase}($field)" + + def node: JsonNode = { + val node = mapper.createObjectNode() + val fieldNode = mapper.createObjectNode() + fieldNode.put("field", field) + node.set(name.toLowerCase(), fieldNode) + node + } } case class MaxTransformAggregation(field: String) extends TransformAggregation { - override def sql: String = s"MAX($field)" - - override def toJson: Map[String, Any] = Map( - "max" -> Map("field" -> field) - ) + override def name: String = "max" } case class MinTransformAggregation(field: String) extends TransformAggregation { - override def sql: String = s"MIN($field)" - - override def toJson: Map[String, Any] = Map( - "min" -> Map("field" -> field) - ) + override def name: String = "min" } case class SumTransformAggregation(field: String) extends TransformAggregation { - override def sql: String = s"SUM($field)" - - override def toJson: Map[String, Any] = Map( - "sum" -> Map("field" -> field) - ) + override def name: String = "sum" } case class AvgTransformAggregation(field: String) extends TransformAggregation { - override def sql: String = s"AVG($field)" - - override def toJson: Map[String, Any] = Map( - "avg" -> Map("field" -> field) - ) + override def name: String = "avg" } case class CountTransformAggregation(field: String) extends TransformAggregation { - override def sql: String = if (field == "_id") "COUNT(*)" else s"COUNT($field)" + override def name: String = "value_count" - override def toJson: Map[String, Any] = Map( - "value_count" -> Map("field" -> field) - ) + override def sql: String = if (field == "_id") "COUNT(*)" else s"COUNT($field)" } case class CardinalityTransformAggregation(field: String) extends TransformAggregation { - override def sql: String = s"COUNT(DISTINCT $field)" + override def name: String = "cardinality" - override def toJson: Map[String, Any] = Map( - "cardinality" -> Map("field" -> field) - ) + override def sql: String = s"COUNT(DISTINCT $field)" } /** Extension methods for AggregateFunction @@ -1105,20 +1138,22 @@ package object schema { case class TransformSync(time: TransformTimeSync) extends DdlToken { override def sql: String = s"SYNC ${time.sql}" - def toJson: Map[String, Any] = { - Map( - "time" -> time.toJson - ) + + def node: JsonNode = { + val node = mapper.createObjectNode() + node.set("time", time.node) + node } } case class TransformTimeSync(field: String, delay: Delay) extends DdlToken { override def sql: String = s"TIME FIELD $field ${delay.sql}" - def toJson: Map[String, Any] = { - Map( - "field" -> field, - "delay" -> delay.toTransformFormat - ) + + def node: JsonNode = { + val node = mapper.createObjectNode() + node.put("field", field) + node.put("delay", delay.toTransformFormat) + node } } @@ -1142,13 +1177,20 @@ package object schema { /** Converts to Elasticsearch JSON format */ - def toJson(implicit criteriaToMap: Criteria => Map[String, Any]): Map[String, Any] = { - Map( - "source" -> source.toJson, - "dest" -> dest.toJson, - "frequency" -> frequency.toTransformFormat, - "sync" -> sync.map(_.toJson) - ) ++ pivot.map(p => "pivot" -> p.toJson) + def node(implicit criteriaToNode: Criteria => JsonNode): JsonNode = { + val node = mapper.createObjectNode() + node.set("source", source.node) + node.set("dest", dest.node) + node.put("frequency", frequency.toTransformFormat) + sync.foreach { s => + node.set("sync", s.node) + () + } + pivot.foreach { p => + node.set("pivot", p.node) + () + } + node } /** SQL DDL representation From cbd6e5d6d30ac61ace82f477d58a86716aff78be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 16 Jan 2026 09:49:28 +0100 Subject: [PATCH 07/22] fix SQL type for COUNT aggr --- .../softnetwork/elastic/sql/function/aggregate/package.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala index a74d02cb..fcf51616 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala @@ -16,6 +16,7 @@ package app.softnetwork.elastic.sql.function +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} import app.softnetwork.elastic.sql.query.{Bucket, BucketPath, Field, Limit, OrderBy, SingleSearch} import app.softnetwork.elastic.sql.{Expr, Identifier, TokenRegex, Updateable} @@ -226,6 +227,8 @@ package object aggregate { partitionBy: Seq[Identifier] = Seq.empty, fields: Seq[Field] = Seq.empty ) extends WindowFunction { + override def baseType: SQLType = SQLTypes.BigInt + def isCardinality: Boolean = identifier.distinct override def limit: Option[Limit] = None From ac4396140d699b017544a67b4f0f4ebb5d91c472 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 16 Jan 2026 09:54:48 +0100 Subject: [PATCH 08/22] update all identifiers within functions - update dependencies --- .../elastic/sql/function/cond/package.scala | 58 +++++++++- .../sql/function/convert/package.scala | 28 ++++- .../elastic/sql/function/math/package.scala | 51 ++++++++- .../elastic/sql/function/package.scala | 13 ++- .../elastic/sql/function/string/package.scala | 103 +++++++++++++++++- .../elastic/sql/function/time/package.scala | 57 +++++++++- .../app/softnetwork/elastic/sql/package.scala | 25 ++--- .../softnetwork/elastic/sql/query/Where.scala | 20 ++-- 8 files changed, 320 insertions(+), 35 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala index 0c432619..f4df7954 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala @@ -17,12 +17,14 @@ package app.softnetwork.elastic.sql.function import app.softnetwork.elastic.sql.{ + query, Expr, Identifier, LiteralParam, PainlessContext, PainlessScript, - TokenRegex + TokenRegex, + Updateable } import app.softnetwork.elastic.sql.`type`.{ SQLAny, @@ -82,6 +84,10 @@ package object cond { s"${arg.trim} == null" // TODO check when identifier is nullable and has functions case _ => throw new IllegalArgumentException("ISNULL requires exactly one argument") } + + override def update(request: query.SingleSearch): IsNull = { + this.copy(identifier = identifier.update(request)) + } } case class IsNotNull(identifier: Identifier) extends ConditionalFunction[SQLAny] { @@ -101,6 +107,10 @@ package object cond { s"${arg.trim} != null" // TODO check when identifier is nullable and has functions case _ => throw new IllegalArgumentException("ISNOTNULL requires exactly one argument") } + + override def update(request: query.SingleSearch): IsNotNull = { + this.copy(identifier = identifier.update(request)) + } } case class Coalesce(values: List[PainlessScript]) @@ -144,6 +154,13 @@ package object cond { } override def nullable: Boolean = values.forall(_.nullable) + + override def update(request: query.SingleSearch): Coalesce = { + this.copy(values = values.map { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case other => other + }) + } } case class NullIf(expr1: PainlessScript, expr2: PainlessScript) @@ -199,6 +216,19 @@ package object cond { _ <- Validator.validateTypesMatching(expr1.out, expr2.out) } yield () } + + override def update(request: query.SingleSearch): NullIf = { + this.copy( + expr1 = expr1 match { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case other => other + }, + expr2 = expr2 match { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case other => other + } + ) + } } case class Case( @@ -211,6 +241,7 @@ package object cond { default.toList override def inputType: SQLAny = SQLTypes.Any + override def outputType: SQLAny = SQLTypes.Any override def sql: String = { @@ -362,6 +393,29 @@ package object cond { override def nullable: Boolean = conditions.exists { case (_, res) => res.nullable } || default.forall(_.nullable) - } + override def update(request: query.SingleSearch): Case = { + this.copy( + expression = expression.map { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case other => other + }, + conditions = conditions.map { case (cond, res) => + val newCond = cond match { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case other => other + } + val newRes = res match { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case other => other + } + (newCond, newRes) + }, + default = default.map { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case other => other + } + ) + } + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala index 8a71bcf4..621106f2 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala @@ -17,13 +17,15 @@ package app.softnetwork.elastic.sql.function import app.softnetwork.elastic.sql.{ + query, Alias, DateMathRounding, Expr, Identifier, PainlessContext, PainlessScript, - TokenRegex + TokenRegex, + Updateable } import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} @@ -100,6 +102,14 @@ package object convert { else s"$Cast($ret)" } value.cast(targetType) + + override def update(request: query.SingleSearch): Conversion = { + value match { + case updatable: Updateable => + this.copy(value = updatable.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case object CastOperator extends Expr("\\:\\:") with TokenRegex @@ -110,6 +120,14 @@ package object convert { override def safe: Boolean = false value.cast(targetType) + + override def update(request: query.SingleSearch): CastOperator = { + value match { + case updatable: Updateable => + this.copy(value = updatable.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case object Convert extends Expr("CONVERT") with TokenRegex @@ -120,5 +138,13 @@ package object convert { override def safe: Boolean = false value.cast(targetType) + + override def update(request: query.SingleSearch): Convert = { + value match { + case updatable: Updateable => + this.copy(value = updatable.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala index adcc4fae..44d2129b 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala @@ -17,12 +17,14 @@ package app.softnetwork.elastic.sql.function import app.softnetwork.elastic.sql.{ + query, Expr, IntValue, PainlessContext, PainlessParam, PainlessScript, - TokenRegex + TokenRegex, + Updateable } import app.softnetwork.elastic.sql.`type`.{SQLNumeric, SQLType, SQLTypes} @@ -91,12 +93,26 @@ package object math { arg: PainlessScript ) extends MathematicalFunction { override def args: List[PainlessScript] = List(arg) + override def update(request: query.SingleSearch): MathematicalFunctionWithOp = { + arg match { + case updatable: Updateable => + this.copy(arg = updatable.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case class Pow(arg: PainlessScript, exponent: Int) extends MathematicalFunction { override def mathOp: MathOp = Pow override def args: List[PainlessScript] = List(arg, IntValue(exponent)) override def nullable: Boolean = arg.nullable + override def update(request: query.SingleSearch): Pow = { + arg match { + case updatable: Updateable => + this.copy(arg = updatable.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case class PowParam(scale: Int) extends PainlessParam with PainlessScript { @@ -130,6 +146,13 @@ package object math { s"${fun.map(_.sql).getOrElse("")}($arg${scale.map(s => s", $s").getOrElse("")})" } + override def update(request: query.SingleSearch): Round = { + arg match { + case updatable: Updateable => + this.copy(arg = updatable.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case class Sign(arg: PainlessScript) extends MathematicalFunction { @@ -143,12 +166,38 @@ package object math { case _ => throw new IllegalArgumentException("Sign function requires exactly one argument") } + override def update(request: query.SingleSearch): Sign = { + arg match { + case updatable: Updateable => + this.copy(arg = updatable.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case class Atan2(y: PainlessScript, x: PainlessScript) extends MathematicalFunction { override def mathOp: MathOp = Atan2 override def args: List[PainlessScript] = List(y, x) override def nullable: Boolean = y.nullable || x.nullable + + override def update(request: query.SingleSearch): Atan2 = { + (y, x) match { + case (updatableY: Updateable, updatableX: Updateable) => + this.copy( + y = updatableY.update(request).asInstanceOf[PainlessScript], + x = updatableX.update(request).asInstanceOf[PainlessScript] + ) + case (updatableY: Updateable, _) => + this.copy( + y = updatableY.update(request).asInstanceOf[PainlessScript] + ) + case (_, updatableX: Updateable) => + this.copy( + x = updatableX.update(request).asInstanceOf[PainlessScript] + ) + case _ => this + } + } } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index ef110b5e..67fb5d4c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -42,7 +42,7 @@ package object function { } } - trait FunctionWithIdentifier extends Function { + trait FunctionWithIdentifier extends Function with Updateable { def identifier: Identifier override def functionNestedElement: Option[NestedElement] = @@ -213,9 +213,18 @@ package object function { override def usesCurrentTimeFunction: Boolean = { functions.exists { _.usesCurrentTimeFunction } } + + override lazy val dependencies: Seq[Identifier] = functions + .foldLeft(Seq.empty[Identifier]) { case (acc, fun) => + acc ++ FunctionUtils.funIdentifiers(fun) + } + .filterNot(_.name.isEmpty) } - trait FunctionN[In <: SQLType, Out <: SQLType] extends Function with PainlessScript { + trait FunctionN[In <: SQLType, Out <: SQLType] + extends Function + with PainlessScript + with Updateable { def fun: Option[PainlessScript] = None def args: List[PainlessScript] diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala index bbd98d01..6c3529cf 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala @@ -16,7 +16,15 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.{Expr, IntValue, PainlessContext, PainlessScript, TokenRegex} +import app.softnetwork.elastic.sql.{ + query, + Expr, + IntValue, + PainlessContext, + PainlessScript, + TokenRegex, + Updateable +} import app.softnetwork.elastic.sql.`type`.{ SQLBigInt, SQLBool, @@ -133,6 +141,12 @@ package object string { extends StringFunction[SQLVarchar] { override def outputType: SQLVarchar = SQLTypes.Varchar override def args: List[PainlessScript] = List(str) + override def update(request: query.SingleSearch): StringFunctionWithOp = { + str match { + case u: Updateable => this.copy(str = u.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case class Substring(str: PainlessScript, start: Int, length: Option[Int]) @@ -172,6 +186,12 @@ package object string { override def toSQL(base: String): String = sql + override def update(request: query.SingleSearch): Substring = { + str match { + case u: Updateable => this.copy(str = u.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case class Concat(values: List[PainlessScript]) extends StringFunction[SQLVarchar] { @@ -182,6 +202,16 @@ package object string { override def nullable: Boolean = values.exists(_.nullable) + override def update(request: query.SingleSearch): Concat = { + values.map { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case v => v + } match { + case updatedValues => this.copy(values = updatedValues) + case _ => this + } + } + override def toPainlessCall( callArgs: List[String], context: Option[PainlessContext] @@ -217,6 +247,12 @@ package object string { override def outputType: SQLBigInt = SQLTypes.BigInt override def stringOp: StringOp = Length override def args: List[PainlessScript] = List(str) + override def update(request: query.SingleSearch): Length = { + str match { + case u: Updateable => this.copy(str = u.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case class LeftFunction(str: PainlessScript, length: Int) extends StringFunction[SQLVarchar] { @@ -245,6 +281,13 @@ package object string { str.validate() override def toSQL(base: String): String = sql + + override def update(request: query.SingleSearch): LeftFunction = { + str match { + case u: Updateable => this.copy(str = u.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case class RightFunction(str: PainlessScript, length: Int) extends StringFunction[SQLVarchar] { @@ -275,6 +318,13 @@ package object string { str.validate() override def toSQL(base: String): String = sql + + override def update(request: query.SingleSearch): RightFunction = { + str match { + case u: Updateable => this.copy(str = u.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case class Replace(str: PainlessScript, search: PainlessScript, replace: PainlessScript) @@ -304,6 +354,21 @@ package object string { } override def toSQL(base: String): String = sql + + override def update(request: query.SingleSearch): Replace = { + List(str, search, replace).map { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case v => v + } match { + case List(updatedStr, updatedSearch, updatedReplace) => + this.copy( + str = updatedStr, + search = updatedSearch, + replace = updatedReplace + ) + case _ => this + } + } } case class Reverse(str: PainlessScript) extends StringFunction[SQLVarchar] { @@ -328,6 +393,13 @@ package object string { str.validate() override def toSQL(base: String): String = sql + + override def update(request: query.SingleSearch): Reverse = { + str match { + case u: Updateable => this.copy(str = u.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case class Position(substr: PainlessScript, str: PainlessScript, start: Int) @@ -360,6 +432,20 @@ package object string { } override def toSQL(base: String): String = sql + + override def update(request: query.SingleSearch): Position = { + List(substr, str).map { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case v => v + } match { + case List(updatedSubstr, updatedStr) => + this.copy( + substr = updatedSubstr, + str = updatedStr + ) + case _ => this + } + } } case class RegexpLike( @@ -394,5 +480,20 @@ package object string { } override def toSQL(base: String): String = sql + + override def update(request: query.SingleSearch): RegexpLike = { + List(str, pattern).map { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case v => v + } match { + case List(updatedStr, updatedPattern) => + this.copy( + str = updatedStr, + pattern = updatedPattern, + matchFlags = matchFlags + ) + case _ => this + } + } } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index 1856002f..a04b77e5 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -17,6 +17,7 @@ package app.softnetwork.elastic.sql.function import app.softnetwork.elastic.sql.{ + query, DateMathRounding, DateMathScript, Expr, @@ -25,7 +26,8 @@ import app.softnetwork.elastic.sql.{ PainlessContext, PainlessScript, StringValue, - TokenRegex + TokenRegex, + Updateable } import app.softnetwork.elastic.sql.operator.time._ import app.softnetwork.elastic.sql.`type`.{ @@ -109,11 +111,13 @@ package object time { case class SQLAddInterval(interval: TimeInterval) extends AddInterval[SQLTemporal] { override def inputType: SQLTemporal = SQLTypes.Temporal override def outputType: SQLTemporal = SQLTypes.Temporal + override def update(request: query.SingleSearch): SQLAddInterval = this } case class SQLSubtractInterval(interval: TimeInterval) extends SubtractInterval[SQLTemporal] { override def inputType: SQLTemporal = SQLTypes.Temporal override def outputType: SQLTemporal = SQLTypes.Temporal + override def update(request: query.SingleSearch): SQLSubtractInterval = this } sealed trait DateTimeFunction extends Function { @@ -283,6 +287,9 @@ package object time { override def shouldBeScripted: Boolean = false + override def update(request: query.SingleSearch): DateTrunc = { + this.copy(identifier = identifier.update(request)) + } } case object Extract extends Expr("EXTRACT") with TokenRegex with PainlessScript { @@ -304,6 +311,7 @@ package object time { override def toSQL(base: String): String = s"$sql(${field.sql} FROM $base)" + override def update(request: query.SingleSearch): Extract = this } import TimeField._ @@ -393,6 +401,9 @@ package object time { } } + override def update(request: query.SingleSearch): LastDayOfMonth = { + this.copy(identifier = identifier.update(request)) + } } case object DateDiff extends Expr("DATE_DIFF") with TokenRegex with PainlessScript { @@ -435,6 +446,21 @@ package object time { } ret } + + override def update(request: query.SingleSearch): DateDiff = { + this.copy( + start = start match { + case updatable: Updateable => + updatable.update(request).asInstanceOf[PainlessScript] + case _ => start + }, + end = end match { + case updatable: Updateable => + updatable.update(request).asInstanceOf[PainlessScript] + case _ => end + } + ) + } } case object DateAdd extends Expr("DATE_ADD") with TokenRegex { @@ -453,6 +479,9 @@ package object time { s"$sql($base, ${interval.sql})" } override def dateMathScript: Boolean = identifier.dateMathScript + override def update(request: query.SingleSearch): DateAdd = { + this.copy(identifier = identifier.update(request)) + } } case object DateSub extends Expr("DATE_SUB") with TokenRegex { @@ -471,6 +500,9 @@ package object time { s"$sql($base, ${interval.sql})" } override def dateMathScript: Boolean = identifier.dateMathScript + override def update(request: query.SingleSearch): DateSub = { + this.copy(identifier = identifier.update(request)) + } } sealed trait FunctionWithDateTimeFormat { @@ -579,6 +611,10 @@ package object time { override def formatScript: Option[String] = Some(format) override def shouldBeScripted: Boolean = true // FIXME + + override def update(request: query.SingleSearch): DateParse = { + this.copy(identifier = identifier.update(request)) + } } case object DateFormat extends Expr("DATE_FORMAT") with TokenRegex with PainlessScript { @@ -625,6 +661,10 @@ package object time { s"$param.format($arg)" case _ => throw new IllegalArgumentException("DateParse requires exactly one argument") } + + override def update(request: query.SingleSearch): DateFormat = { + this.copy(identifier = identifier.update(request)) + } } case object DateTimeAdd extends Expr("DATETIME_ADD") with TokenRegex { @@ -643,6 +683,10 @@ package object time { s"$sql($base, ${interval.sql})" } override def dateMathScript: Boolean = identifier.dateMathScript + + override def update(request: query.SingleSearch): DateTimeAdd = { + this.copy(identifier = identifier.update(request)) + } } case object DateTimeSub extends Expr("DATETIME_SUB") with TokenRegex { @@ -661,6 +705,10 @@ package object time { s"$sql($base, ${interval.sql})" } override def dateMathScript: Boolean = identifier.dateMathScript + + override def update(request: query.SingleSearch): DateTimeSub = { + this.copy(identifier = identifier.update(request)) + } } case object DateTimeParse extends Expr("DATETIME_PARSE") with TokenRegex with PainlessScript { @@ -720,6 +768,10 @@ package object time { } override def formatScript: Option[String] = Some(format) + + override def update(request: query.SingleSearch): DateTimeParse = { + this.copy(identifier = identifier.update(request)) + } } case object DateTimeFormat extends Expr("DATETIME_FORMAT") with TokenRegex with PainlessScript { @@ -769,6 +821,9 @@ package object time { case _ => throw new IllegalArgumentException("DateParse requires exactly one argument") } + override def update(request: query.SingleSearch): DateTimeFormat = { + this.copy(identifier = identifier.update(request)) + } } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index ccd399ca..353d0e3f 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -1101,8 +1101,7 @@ package object sql { nestedElement: Option[NestedElement] = None, bucketPath: String = "", col: Option[Column] = None, - table: Option[String] = None, - override val dependencies: Seq[Identifier] = Seq.empty + table: Option[String] = None ) extends Identifier { def withFunctions(functions: List[Function]): Identifier = this.copy(functions = functions) @@ -1131,12 +1130,7 @@ package object sql { } val parts: Seq[String] = name.split("\\.").toSeq val tableAlias = parts.head - val table = request.tableAliases.find(t => t._2 == tableAlias).map(_._2) - val dependencies = functions - .foldLeft(Seq.empty[Identifier]) { case (acc, fun) => - acc ++ FunctionUtils.funIdentifiers(fun) - } - .filterNot(_.name.isEmpty) + val table = request.tableAliases.find(t => t._2 == tableAlias).map(_._1) if (table.nonEmpty) { request.unnestAliases.find(_._1 == tableAlias) match { case Some(tuple) if !nested => @@ -1157,8 +1151,7 @@ package object sql { nestedElement = nestedElement, bucketPath = bucketPath, col = request.schema.flatMap(schema => schema.find(colName)), - table = table, - dependencies = dependencies.map(_.update(request)) + table = table ) .withFunctions(this.updateFunctions(request)) case Some(tuple) if nested => @@ -1172,8 +1165,7 @@ package object sql { bucket = request.bucketNames.get(identifierName).orElse(bucket), bucketPath = bucketPath, col = request.schema.flatMap(schema => schema.find(colName)), - table = table, - dependencies = dependencies.map(_.update(request)) + table = table ) .withFunctions(this.updateFunctions(request)) case None if nested => @@ -1184,8 +1176,7 @@ package object sql { bucket = request.bucketNames.get(identifierName).orElse(bucket), bucketPath = bucketPath, col = request.schema.flatMap(schema => schema.find(name)), - table = table, - dependencies = dependencies.map(_.update(request)) + table = table ) .withFunctions(this.updateFunctions(request)) case _ => @@ -1197,8 +1188,7 @@ package object sql { bucket = request.bucketNames.get(identifierName).orElse(bucket), bucketPath = bucketPath, col = request.schema.flatMap(schema => schema.find(colName)), - table = table, - dependencies = dependencies.map(_.update(request)) + table = table ) } } else { @@ -1207,8 +1197,7 @@ package object sql { fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias), bucket = request.bucketNames.get(identifierName).orElse(bucket), bucketPath = bucketPath, - col = request.schema.flatMap(schema => schema.find(name)), - dependencies = dependencies.map(_.update(request)) + col = request.schema.flatMap(schema => schema.find(name)) ) .withFunctions(this.updateFunctions(request)) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index 58cffbfd..ddd4acc4 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -31,11 +31,11 @@ case object Where extends Expr("WHERE") with TokenRegex sealed trait Criteria extends Updateable with PainlessScript { def operator: Operator - def identifiers: Seq[Identifier] = this match { - case Predicate(left, _, right, _, _) => left.identifiers ++ right.identifiers - case c: Expression => c.identifiers - case relation: ElasticRelation => relation.criteria.identifiers - case m: MultiMatchCriteria => m.identifiers + def dependencies: Seq[Identifier] = this match { + case Predicate(left, _, right, _, _) => left.dependencies ++ right.dependencies + case c: Expression => c.dependencies + case relation: ElasticRelation => relation.criteria.dependencies + case m: MultiMatchCriteria => m.dependencies case _ => Nil } @@ -285,10 +285,10 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { def valueAsString: String = maybeValue.map(v => s" $v").getOrElse("") override def sql = s"$identifier $notAsString$operator$valueAsString" - override def identifiers: Seq[Identifier] = + override lazy val dependencies: Seq[Identifier] = maybeValue match { - case Some(id: Identifier) => Seq(identifier, id) - case _ => Seq(identifier) + case Some(id: Identifier) => identifier.dependencies ++ id.dependencies + case _ => identifier.dependencies } override def extractAllMetricsPath: Map[String, String] = @@ -768,7 +768,7 @@ case class DistanceCriteria( } case class MultiMatchCriteria( - override val identifiers: Seq[Identifier], + identifiers: Seq[Identifier], value: StringValue, nestedElement: Option[NestedElement] = None ) extends Criteria { // FIXME map to multi_match @@ -778,6 +778,8 @@ case class MultiMatchCriteria( override def update(request: SingleSearch): Criteria = this.copy(identifiers = identifiers.map(_.update(request))) + override def dependencies: Seq[Identifier] = identifiers.flatMap(_.dependencies) + override lazy val nested: Boolean = identifiers.forall(_.nested) @tailrec From b433a685d71c677f03b0b4a5cbd8aab64fda385e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 16 Jan 2026 09:57:56 +0100 Subject: [PATCH 09/22] add naming utils, add isObject flag --- .../softnetwork/elastic/schema/package.scala | 72 +++++++++++++++++++ .../app/softnetwork/elastic/sql/package.scala | 33 +++++---- .../elastic/sql/query/GroupBy.scala | 2 + .../elastic/sql/query/Select.scala | 4 +- 4 files changed, 93 insertions(+), 18 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index bb27f971..9e8cfb9a 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -506,4 +506,76 @@ package object schema { ) } } + + object NamingUtils { + + /** Normalizes Elasticsearch object names (indices, pipelines, transforms, policies) + * + * Rules: + * - Only lowercase alphanumeric + underscore + dot + * - Max 255 characters + * - If too long, truncates with readable prefix + hash suffix + */ + def normalizeObjectName(name: String, maxLength: Int = 255): String = { + require(maxLength > 10, "maxLength must be > 10 to allow prefix + hash") + + // Step 1: Normalize to Elasticsearch-safe characters + // Keep: a-z, 0-9, underscore, dot + val normalized = name.toLowerCase + .replaceAll("[^a-zA-Z0-9_.]", "_") // caractères invalides -> "_" + .replaceAll("_+", "_") // compacter plusieurs "_" + .stripPrefix("_") + .stripSuffix("_") + + // Step 2: If within limit, return as-is + if (normalized.length <= maxLength) { + return normalized + } + + // Step 3: Generate readable truncated name with hash + // Format: _ + // Example: orders_with_customers_mv_orders_enri_a1b2c3d4 + + val hashLength = 8 // Short hash for readability + val prefixLength = maxLength - hashLength - 1 // -1 for underscore + + val prefix = normalized.take(prefixLength) + val hash = generateShortHash(normalized, hashLength) + + s"${prefix}_${hash}" + } + + /** Generates a short deterministic hash from a string + * + * Uses MD5 for consistency, takes first N hex chars + */ + private def generateShortHash(input: String, length: Int = 8): String = { + val digest = MessageDigest.getInstance("MD5") + val hashBytes = digest.digest(input.getBytes("UTF-8")) + hashBytes.map("%02x".format(_)).mkString.take(length) + } + + /** Validates an Elasticsearch object name + * + * @return + * Right(name) if valid, Left(error) otherwise + */ + def validateObjectName(name: String): Either[String, String] = { + if (name.isEmpty) { + Left("Name cannot be empty") + } else if (name.length > 255) { + Left(s"Name exceeds 255 characters: ${name.length}") + } else if (!name.matches("^[a-z0-9_.]+$")) { + Left( + s"Name contains invalid characters. Only lowercase alphanumeric, underscore and dot allowed: $name" + ) + } else if (name.startsWith("_") || name.startsWith("-") || name.startsWith("+")) { + Left(s"Name cannot start with _, - or +: $name") + } else if (name == "." || name == "..") { + Left("Name cannot be . or ..") + } else { + Right(name) + } + } + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 353d0e3f..cdf86034 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -16,6 +16,7 @@ package app.softnetwork.elastic +import app.softnetwork.elastic.schema.NamingUtils import app.softnetwork.elastic.sql.function.aggregate.{AggregateFunction, COUNT, WindowFunction} import app.softnetwork.elastic.sql.function.geo.DistanceUnit import app.softnetwork.elastic.sql.function.time.CurrentFunction @@ -325,7 +326,7 @@ package object sql { case f: Float => FloatValue(f) case d: Double => DoubleValue(d) case a: Array[T] => apply(a.toSeq) - case a: Seq[T] => + case a: Seq[T] if a.nonEmpty => val values = a.map(apply) values.headOption match { case Some(_: StringValue) => @@ -352,6 +353,7 @@ package object sql { ).asInstanceOf[Values[R, T]] case _ => throw new IllegalArgumentException("Unsupported Values type") } + case _: Seq[T] => EmptyValues().asInstanceOf[Values[R, T]] case o: Map[_, _] => val map = o.asInstanceOf[Map[String, Any]].map { case (k, v) => k -> apply(v) } ObjectValue(map) @@ -770,6 +772,11 @@ package object sql { def toJson: JsonNode = this } + case class EmptyValues() extends Values[Any, Null](Seq.empty) { + override def sql: String = "()" + override def nullable: Boolean = true + } + def toRegex(value: String): String = { value.replaceAll("%", ".*").replaceAll("_", ".") } @@ -795,22 +802,7 @@ package object sql { acc.replace(k, s"_${v}_") } // Nettoyer pour obtenir un identifiant valide - val normalized = replaced - .replaceAll("[^a-zA-Z0-9_]", "_") // caractères invalides -> "_" - .replaceAll("_+", "_") // compacter plusieurs "_" - .stripPrefix("_") - .stripSuffix("_") - .toLowerCase - - // Tronquer si nécessaire - if (normalized.length > MaxAliasLength) { - val digest = MessageDigest.getInstance("MD5").digest(normalized.getBytes("UTF-8")) - val hash = digest.map("%02x".format(_)).mkString.take(MaxAliasLength) - //normalized.take(MaxAliasLength - hash.length - 1) + "_" + hash - hash - } else { - normalized - } + NamingUtils.normalizeObjectName(replaced, MaxAliasLength).replaceAll("\\.", "_") } } @@ -1078,6 +1070,13 @@ package object sql { def isWindowing: Boolean = windows.exists(_.partitionBy.nonEmpty) def painlessScriptRequired: Boolean = functions.nonEmpty && !hasAggregation && bucket.isEmpty + + def isObject: Boolean = { + out match { + case SQLTypes.Struct => true + case _ => name.contains(".") + } + } } object Identifier { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala index 4437309d..16ded040 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala @@ -117,6 +117,8 @@ case class Bucket( */ override def painless(context: Option[PainlessContext]): String = identifier.painless(context) + + def isObject: Boolean = identifier.isObject } case class BucketPath(buckets: Seq[Bucket]) { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala index 8fbf774f..e826241f 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala @@ -50,7 +50,7 @@ case class Field( def isScriptField: Boolean = identifier.painlessScriptRequired override def sql: String = s"$identifier${asString(fieldAlias)}" - override def dependencies: Seq[Identifier] = identifier.dependencies + override lazy val dependencies: Seq[Identifier] = identifier.dependencies lazy val sourceField: String = { if (identifier.nested) { @@ -109,6 +109,8 @@ case class Field( lazy val path: String = identifier.path def isBucketScript: Boolean = !isAggregation && hasAggregation + + def isObject: Boolean = identifier.isObject } case object Except extends Expr("except") with TokenRegex From 8ccd0d341c4bfeca2598c55cdebc87840bf7e297 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 16 Jan 2026 10:01:42 +0100 Subject: [PATCH 10/22] use painless transform param if painless context is transform --- .../app/softnetwork/elastic/sql/package.scala | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index cdf86034..e3e62aaf 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -25,7 +25,6 @@ import app.softnetwork.elastic.sql.query._ import app.softnetwork.elastic.sql.schema.Column import com.fasterxml.jackson.databind.JsonNode -import java.security.MessageDigest import scala.annotation.tailrec import scala.jdk.CollectionConverters._ import scala.reflect.runtime.universe._ @@ -156,6 +155,8 @@ package object sql { def isProcessor: Boolean = context == PainlessContextType.Processor + def isTransform: Boolean = context == PainlessContextType.Transform + lazy val timestamp: String = { context match { case PainlessContextType.Processor => CurrentFunction.processorTimestamp @@ -174,9 +175,19 @@ package object sql { def addParam(token: Token): Option[String] = { token match { case identifier: Identifier if isProcessor => - addParam( - LiteralParam(identifier.processParamName, None /*identifier.processCheckNotNull*/ ) - ) + if (identifier.name.nonEmpty) + addParam( + LiteralParam(identifier.processParamName, None /*identifier.processCheckNotNull*/ ) + ) + else + None + case identifier: Identifier if isTransform => + if (identifier.name.nonEmpty) + addParam( + LiteralParam(identifier.transformParamName, None /*identifier.processCheckNotNull*/ ) + ) + else + None case param: PainlessParam if param.param.nonEmpty && (param.isInstanceOf[LiteralParam] || param.nullable) => get(param) match { @@ -985,7 +996,23 @@ package object sql { lazy val processCheckNotNull: Option[String] = if (path.isEmpty || !nullable) None else - Option(s"(ctx.$path == null ? $nullValue : ctx.$path${painlessMethods.mkString("")})") + Option( + s"($processParamName == null ? $nullValue : $processParamName${painlessMethods.mkString("")})" + ) + + lazy val transformParamName: String = { + fieldAlias match { + case Some(a) => s"ctx.$a" + case None => processParamName + } + } + + lazy val transformCheckNotNull: Option[String] = + if (path.isEmpty || !nullable) None + else + Option( + s"($transformParamName == null ? $nullValue : $transformParamName${painlessMethods.mkString("")})" + ) def originalType: SQLType = if (name.trim.nonEmpty) SQLTypes.Any From 57252da0b93f362cf45f942aedd979655e738af9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 16 Jan 2026 10:49:47 +0100 Subject: [PATCH 11/22] update transform param name, update painless context creation using implicit context type --- .../elastic/sql/bridge/ElasticAggregation.scala | 6 +++--- .../softnetwork/elastic/sql/bridge/package.scala | 10 +++++----- .../elastic/sql/bridge/ElasticAggregation.scala | 6 +++--- .../softnetwork/elastic/sql/bridge/package.scala | 10 +++++----- .../scala/app/softnetwork/elastic/sql/package.scala | 13 +++++++++---- 5 files changed, 25 insertions(+), 20 deletions(-) diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index 7f0217d4..0c303fa1 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -156,14 +156,14 @@ object ElasticAggregation { buildScript: (String, Script) => Aggregation ): Aggregation = { if (transformFuncs.nonEmpty) { - val context = PainlessContext() + val context = PainlessContext(context = contextType) val scriptSrc = identifier.painless(Some(context)) val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) } else { aggType match { case th: WindowFunction if th.shouldBeScripted => - val context = PainlessContext() + val context = PainlessContext(context = contextType) val scriptSrc = th.identifier.painless(Some(context)) val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) @@ -370,7 +370,7 @@ object ElasticAggregation { val aggScript = if (!bucket.isBucketScript && bucket.shouldBeScripted) { - val context = PainlessContext() + val context = PainlessContext(context = contextType) val painless = bucket.painless(Some(context)) Some(now(Script(s"$context$painless").lang("painless"))) } else { diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 25c1903e..9da37706 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -517,7 +517,7 @@ package object bridge { case Nil => _search case _ => _search scriptfields scriptFields.map { field => - val context = PainlessContext() + val context = PainlessContext(context = contextType) val script = field.painless(Some(context)) scriptField( field.scriptName, @@ -538,7 +538,7 @@ package object bridge { case Some(o) if aggregates.isEmpty && buckets.isEmpty => _search sortBy o.sorts.map { sort => if (sort.isScriptSort) { - val context = PainlessContext() + val context = PainlessContext(context = contextType) val painless = sort.field.painless(Some(context)) val painlessScript = s"$context$painless" val script = @@ -624,7 +624,7 @@ package object bridge { case _ => true })) ) { - val context = PainlessContext() + val context = PainlessContext(context = contextType) val script = painless(Some(context)) return scriptQuery( now(Script(script = s"$context$script").lang("painless").scriptType("source")) @@ -842,7 +842,7 @@ package object bridge { case NE | DIFF => not(rangeQuery(identifier.name) gte script lte script) } case _ => - val context = PainlessContext() + val context = PainlessContext(context = contextType) val script = painless(Some(context)) scriptQuery( now( @@ -853,7 +853,7 @@ package object bridge { ) } case _ => - val context = PainlessContext() + val context = PainlessContext(context = contextType) val script = painless(Some(context)) scriptQuery( now( diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index b7535b01..e6cacfa2 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -156,14 +156,14 @@ object ElasticAggregation { buildScript: (String, Script) => Aggregation ): Aggregation = { if (transformFuncs.nonEmpty) { - val context = PainlessContext() + val context = PainlessContext(context = contextType) val scriptSrc = identifier.painless(Some(context)) val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) } else { aggType match { case th: WindowFunction if th.shouldBeScripted => - val context = PainlessContext() + val context = PainlessContext(context = contextType) val scriptSrc = th.identifier.painless(Some(context)) val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) @@ -370,7 +370,7 @@ object ElasticAggregation { val aggScript = if (!bucket.isBucketScript && bucket.shouldBeScripted) { - val context = PainlessContext() + val context = PainlessContext(context = contextType) val painless = bucket.painless(Some(context)) Some(now(Script(s"$context$painless").lang("painless"))) } else { diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 09711ca8..2f22881a 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -512,7 +512,7 @@ package object bridge { case Nil => _search case _ => _search scriptfields scriptFields.map { field => - val context = PainlessContext() + val context = PainlessContext(context = contextType) val script = field.painless(Some(context)) scriptField( field.scriptName, @@ -533,7 +533,7 @@ package object bridge { case Some(o) if aggregates.isEmpty && buckets.isEmpty => _search sortBy o.sorts.map { sort => if (sort.isScriptSort) { - val context = PainlessContext() + val context = PainlessContext(context = contextType) val painless = sort.field.painless(Some(context)) val painlessScript = s"$context$painless" val script = @@ -619,7 +619,7 @@ package object bridge { case _ => true })) ) { - val context = PainlessContext() + val context = PainlessContext(context = contextType) val script = painless(Some(context)) return scriptQuery( now( @@ -841,7 +841,7 @@ package object bridge { case NE | DIFF => not(rangeQuery(identifier.name) gte script lte script) } case _ => - val context = PainlessContext() + val context = PainlessContext(context = contextType) val script = painless(Some(context)) scriptQuery( now( @@ -852,7 +852,7 @@ package object bridge { ) } case _ => - val context = PainlessContext() + val context = PainlessContext(context = contextType) val script = painless(Some(context)) scriptQuery( now( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index e3e62aaf..8c0ddc39 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -184,7 +184,7 @@ package object sql { case identifier: Identifier if isTransform => if (identifier.name.nonEmpty) addParam( - LiteralParam(identifier.transformParamName, None /*identifier.processCheckNotNull*/ ) + LiteralParam(identifier.transformParamName, identifier.transformCheckNotNull) ) else None @@ -212,6 +212,10 @@ package object sql { token match { case identifier: Identifier if isProcessor => get(LiteralParam(identifier.processParamName, None /*identifier.processCheckNotNull*/ )) + case identifier: Identifier if isTransform => + get( + LiteralParam(identifier.transformParamName, None /*identifier.transformCheckNotNull*/ ) + ) case param: PainlessParam => if (exists(param)) Try(_values(_keys.indexOf(param))).toOption else None @@ -1002,8 +1006,8 @@ package object sql { lazy val transformParamName: String = { fieldAlias match { - case Some(a) => s"ctx.$a" - case None => processParamName + case Some(a) => s"doc['$a'].value" + case None => paramName } } @@ -1011,7 +1015,8 @@ package object sql { if (path.isEmpty || !nullable) None else Option( - s"($transformParamName == null ? $nullValue : $transformParamName${painlessMethods.mkString("")})" + s"(doc['$transformParamName'].size() == 0 ? $nullValue : doc['$transformParamName'].value${painlessMethods + .mkString("")})" ) def originalType: SQLType = From 082e1f2c1e94583e29d25d80ad317b4a2fab3dd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 16 Jan 2026 10:58:05 +0100 Subject: [PATCH 12/22] fix transform param name --- .../scala/app/softnetwork/elastic/sql/package.scala | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 8c0ddc39..1dcda092 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -1004,12 +1004,11 @@ package object sql { s"($processParamName == null ? $nullValue : $processParamName${painlessMethods.mkString("")})" ) - lazy val transformParamName: String = { - fieldAlias match { - case Some(a) => s"doc['$a'].value" - case None => paramName - } - } + lazy val transformParamName: String = + if (isAggregation && functions.size == 1) s"params.${metricName.getOrElse(aliasOrName)}" + else if (aliasOrName.nonEmpty) + s"doc['$aliasOrName'].value" + else "" lazy val transformCheckNotNull: Option[String] = if (path.isEmpty || !nullable) None From f0b532053eda28b99deefbdb20523b00b5639299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 16 Jan 2026 11:53:40 +0100 Subject: [PATCH 13/22] compute bucket selector script within having, add LatestValueTransformAggregation, update TransformBucketSelectorConfig and TransformPivot --- .../elastic/sql/query/Having.scala | 10 ++ .../elastic/sql/schema/package.scala | 157 +++++++++++++++++- 2 files changed, 161 insertions(+), 6 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala index 243116f7..9ccd1b42 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala @@ -32,4 +32,14 @@ case class Having(criteria: Option[Criteria]) extends Updateable { def nestedElements: Seq[NestedElement] = criteria.map(_.nestedElements).getOrElse(Seq.empty).groupBy(_.path).map(_._2.head).toList + + def script: Option[String] = criteria.flatMap { criteria => + val fullScript = MetricSelectorScript + .metricSelector(criteria) + .replaceAll("1 == 1 &&", "") + .replaceAll("&& 1 == 1", "") + .replaceAll("1 == 1", "") + .trim + if (fullScript.nonEmpty) Some(fullScript) else None + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 96a40a36..16512c40 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -732,6 +732,29 @@ package object schema { def sql: String = s"CREATE OR REPLACE ENRICH POLICY $name TYPE $policyType WITH SOURCE INDICES ${indices .mkString(",")} MATCH FIELD $matchField ENRICH FIELDS (${enrichFields.mkString(", ")})${Where(criteria)}" + + def node(implicit criteriaToNode: Criteria => JsonNode): JsonNode = { + val node = mapper.createObjectNode() + node.put("name", name) + node.put("type", policyType.name) + val indicesNode = mapper.createArrayNode() + indices.foreach { index => + indicesNode.add(index) + () + } + node.set("indices", indicesNode) + node.put("match_field", matchField) + val enrichFieldsNode = mapper.createArrayNode() + enrichFields.foreach { field => + enrichFieldsNode.add(field) + () + } + node.set("enrich_fields", enrichFieldsNode) + criteria.foreach { c => + node.set("query", implicitly[JsonNode](c)) + } + node + } } // ==================== Transform ==================== @@ -974,9 +997,9 @@ package object schema { case class TransformBucketSelectorConfig( name: String = "having_filter", bucketsPath: Map[String, String], - having: Criteria + script: String ) { - def node(implicit criteriaToNode: Criteria => JsonNode): JsonNode = { + def node: JsonNode = { val node = mapper.createObjectNode() val bucketSelectorNode = mapper.createObjectNode() val bucketsPathNode = mapper.createObjectNode() @@ -985,7 +1008,9 @@ package object schema { () } bucketSelectorNode.set("buckets_path", bucketsPathNode) - bucketSelectorNode.set("script", implicitly[JsonNode](having)) + if (script.nonEmpty) { + bucketSelectorNode.put("script", script) + } node.set("bucket_selector", bucketSelectorNode) node } @@ -994,7 +1019,8 @@ package object schema { case class TransformPivot( groupBy: Map[String, TransformGroupBy], aggregations: Map[String, TransformAggregation], - bucketSelector: Option[TransformBucketSelectorConfig] = None + bucketSelector: Option[TransformBucketSelectorConfig] = None, + script: Option[String] = None ) extends DdlToken { override def sql: String = { val groupByStr = groupBy @@ -1009,14 +1035,14 @@ package object schema { } .mkString(", ") - val havingStr = bucketSelector.map(bs => s" HAVING ${bs.having}").getOrElse("") + val havingStr = bucketSelector.map(bs => s" HAVING ${bs.script}").getOrElse("") s"PIVOT ($groupByStr) AGGREGATE ($aggStr)$havingStr" } /** Converts to JSON for Elasticsearch */ - def node(implicit criteriaToNode: Criteria => JsonNode): JsonNode = { + def node: JsonNode = { val node = mapper.createObjectNode() val groupByNode = mapper.createObjectNode() @@ -1136,6 +1162,125 @@ package object schema { } } + /** Captures the latest value for a field (for changelog snapshots) + * + * Uses top_hits with size=1 sorted by a timestamp field This works for ANY field type (text, + * numeric, date, etc.) + */ + case class LatestValueTransformAggregation( + field: String, + sortBy: String = "_ingest.timestamp" // Default sort by ingest timestamp + ) extends TransformAggregation { + + override def name: String = "latest_value" + + override def sql: String = s"LAST_VALUE($field)" + + override def node: JsonNode = { + val node = mapper.createObjectNode() + val topHitsNode = mapper.createObjectNode() + + // Configure top_hits to get the latest value + topHitsNode.put("size", 1) + + // Sort by timestamp descending to get latest + val sortArray = mapper.createArrayNode() + val sortObj = mapper.createObjectNode() + val sortFieldObj = mapper.createObjectNode() + sortFieldObj.put("order", "desc") + sortObj.set(sortBy, sortFieldObj) + sortArray.add(sortObj) + topHitsNode.set("sort", sortArray) + + // Only retrieve the field we need + val sourceObj = mapper.createObjectNode() + val includesArray = mapper.createArrayNode() + includesArray.add(field) + sourceObj.set("includes", includesArray) + topHitsNode.set("_source", sourceObj) + + node.set("top_hits", topHitsNode) + node + } + } + + /** Alternative: Use MAX aggregation for compatible types + * + * This is simpler but only works for: + * - Numeric fields (int, long, double, float) + * - Date fields + * - Keyword fields (lexicographic max) + */ + case class MaxValueTransformAggregation(field: String) extends TransformAggregation { + override def name: String = "max" + + override def sql: String = s"MAX($field)" + } + + /** Alternative: Use MIN aggregation (for specific use cases) + */ + case class MinValueTransformAggregation(field: String) extends TransformAggregation { + override def name: String = "min" + + override def sql: String = s"MIN($field)" + } + + object ChangelogAggregationStrategy { + + /** Selects the appropriate aggregation for a field in a changelog based on its data type + * + * @param field + * The field to aggregate + * @param dataType + * The SQL data type of the field + * @return + * The transform aggregation to use + */ + def selectAggregation(field: String, dataType: SQLType): TransformAggregation = { + dataType match { + // Numeric types: Use MAX directly + case SQLTypes.Int | SQLTypes.BigInt | SQLTypes.Double | SQLTypes.Real => + MaxValueTransformAggregation(field) + + // Date/Timestamp: Use MAX directly + case SQLTypes.Date | SQLTypes.Timestamp => + MaxValueTransformAggregation(field) + + // Boolean: Use MAX directly (1=true, 0=false) + case SQLTypes.Boolean => + MaxValueTransformAggregation(field) + + // Keyword: Already a keyword type, use MAX directly + case SQLTypes.Keyword => + MaxValueTransformAggregation(field) // ✅ NO .keyword suffix + + // Text/Varchar: Analyzed field, must use .keyword multi-field + case SQLTypes.Text | SQLTypes.Varchar => + MaxValueTransformAggregation(s"$field.keyword") // ✅ Use .keyword multi-field + + // For complex types, use top_hits as fallback + case _ => + LatestValueTransformAggregation(field) + } + } + + /** Determines if a field can use simple MAX aggregation (without needing a multi-field) + */ + def canUseMaxDirectly(dataType: SQLType): Boolean = dataType match { + case SQLTypes.Int | SQLTypes.BigInt | SQLTypes.Double | SQLTypes.Real | SQLTypes.Date | + SQLTypes.Timestamp | SQLTypes.Keyword | SQLTypes.Boolean => + true + case _ => false + } + + /** Determines if a field needs a .keyword multi-field for aggregation + */ + def needsKeywordMultiField(dataType: SQLType): Boolean = dataType match { + case SQLTypes.Text | SQLTypes.Varchar => true + case _ => false + } + } + case class TransformSync(time: TransformTimeSync) extends DdlToken { override def sql: String = s"SYNC ${time.sql}" From 1824bd2e94e0c4ba81d9c66aae644e0c38cfbf6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 16 Jan 2026 18:17:46 +0100 Subject: [PATCH 14/22] init licensing module --- build.sbt | 12 ++ .../elastic/client/GatewayApi.scala | 58 ++++++++- .../client/licensing/DdlLicenseChecker.scala | 73 +++++++++++ .../client/licensing/DqlLicenseChecker.scala | 64 ++++++++++ licensing/build.sbt | 4 + .../licensing/DefaultLicenseManager.scala | 78 ++++++++++++ .../elastic/licensing/LicenseChecker.scala | 60 +++++++++ .../elastic/licensing/package.scala | 116 ++++++++++++++++++ .../softnetwork/elastic/schema/package.scala | 48 +++++++- .../elastic/sql/parser/Parser.scala | 18 ++- .../elastic/sql/query/package.scala | 53 +++++++- .../elastic/sql/schema/package.scala | 62 +++++++++- .../elastic/sql/parser/ParserSpec.scala | 35 +++++- 13 files changed, 661 insertions(+), 20 deletions(-) create mode 100644 core/src/main/scala/app/softnetwork/elastic/client/licensing/DdlLicenseChecker.scala create mode 100644 core/src/main/scala/app/softnetwork/elastic/client/licensing/DqlLicenseChecker.scala create mode 100644 licensing/build.sbt create mode 100644 licensing/src/main/scala/app/softnetwork/elastic/licensing/DefaultLicenseManager.scala create mode 100644 licensing/src/main/scala/app/softnetwork/elastic/licensing/LicenseChecker.scala create mode 100644 licensing/src/main/scala/app/softnetwork/elastic/licensing/package.scala diff --git a/build.sbt b/build.sbt index 7fe63779..8ad82d69 100644 --- a/build.sbt +++ b/build.sbt @@ -99,6 +99,14 @@ ThisBuild / libraryDependencySchemes += "org.scala-lang.modules" %% "scala-xml" Test / parallelExecution := false +lazy val licensing = project + .in(file("licensing")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + moduleSettings + ) + lazy val sql = project .in(file("sql")) .configs(IntegrationTest) @@ -142,6 +150,9 @@ lazy val core = project .dependsOn( macros % "compile->compile;test->test;it->it" ) + .dependsOn( + licensing % "compile->compile;test->test;it->it" + ) lazy val persistence = project .in(file("persistence")) @@ -461,6 +472,7 @@ lazy val root = project crossScalaVersions := Nil ) .aggregate( + licensing, sql, bridge, macros, diff --git a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala index 69b8f80c..70652228 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala @@ -17,6 +17,7 @@ package app.softnetwork.elastic.client import akka.actor.ActorSystem +import app.softnetwork.elastic.client.licensing.{DdlLicenseChecker, DqlLicenseChecker} import app.softnetwork.elastic.client.result.{ DdlResult, DmlResult, @@ -32,6 +33,7 @@ import app.softnetwork.elastic.client.result.{ SQLResult, TableResult } +import app.softnetwork.elastic.licensing.{DefaultLicenseManager, LicenseChecker, LicenseManager} import app.softnetwork.elastic.sql.parser.Parser import app.softnetwork.elastic.sql.query.{ AlterTable, @@ -1165,6 +1167,24 @@ trait GatewayApi extends ElasticClientHelpers { with ScrollApi with VersionApi => + // ✅ Inject license manager (overridable) + def licenseManager: LicenseManager = new DefaultLicenseManager() + + // ✅ Create checkers (depend on licenseManager) + def dqlChecker(implicit + system: ActorSystem + ): LicenseChecker[DqlStatement, ElasticResult[QueryResult]] = + new DqlLicenseChecker(licenseManager, logger) + + def ddlChecker(implicit + system: ActorSystem + ): LicenseChecker[DdlStatement, ElasticResult[QueryResult]] = + new DdlLicenseChecker( + licenseManager, + logger, + () => countMaterializedViews() + ) + lazy val dqlExecutor = new DqlExecutor( api = this, logger = logger @@ -1190,6 +1210,13 @@ trait GatewayApi extends ElasticClientHelpers { tableExec = tableExecutor ) + // ✅ Helper to count materialized views + private[client] def countMaterializedViews()(implicit system: ActorSystem): Future[Int] = { + implicit val ec: ExecutionContext = system.dispatcher + // Query Elasticsearch for indices with _meta.type = "materialized_view" + Future.successful(0) // TODO: Implement + } + // ======================================================================== // SQL GATEWAY API // ======================================================================== @@ -1255,18 +1282,41 @@ trait GatewayApi extends ElasticClientHelpers { def run( statement: Statement )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + statement match { + // ✅ DQL with license check case dql: DqlStatement => - dqlExecutor.execute(dql) + dqlChecker.check(dql, dqlExecutor.execute).map { + case Right(result) => result + case Left(licenseError) => + ElasticFailure( + ElasticError( + message = licenseError.message, + statusCode = Some(licenseError.statusCode), + operation = Some("license") + ) + ) + } - // handle DML statements + // ✅ DML (no license check for now) case dml: DmlStatement => dmlExecutor.execute(dml) - // handle DDL statements + // ✅ DDL with license check case ddl: DdlStatement => - ddlExecutor.execute(ddl) + ddlChecker.check(ddl, ddlExecutor.execute).map { + case Right(result) => result + case Left(licenseError) => + ElasticFailure( + ElasticError( + message = licenseError.message, + statusCode = Some(licenseError.statusCode), + operation = Some("license") + ) + ) + } case _ => // unsupported SQL statement diff --git a/core/src/main/scala/app/softnetwork/elastic/client/licensing/DdlLicenseChecker.scala b/core/src/main/scala/app/softnetwork/elastic/client/licensing/DdlLicenseChecker.scala new file mode 100644 index 00000000..b9e745d8 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/licensing/DdlLicenseChecker.scala @@ -0,0 +1,73 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.licensing + +import akka.actor.ActorSystem +import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.licensing._ +import app.softnetwork.elastic.sql.query._ +import org.slf4j.Logger + +import scala.concurrent.{ExecutionContext, Future} + +/** License checker for DDL statements. + */ +class DdlLicenseChecker( + licenseManager: LicenseManager, + logger: Logger, + mvCounter: () => Future[Int] // ✅ Dependency injection +)(implicit system: ActorSystem) + extends LicenseChecker[DdlStatement, ElasticResult[QueryResult]] { + + implicit val ec: ExecutionContext = system.dispatcher + + override def check( + request: DdlStatement, + execute: DdlStatement => Future[ElasticResult[QueryResult]] + ): Future[Either[LicenseError, ElasticResult[QueryResult]]] = { + + request match { + case create: CreateMaterializedView => + // Check feature availability + if (!licenseManager.hasFeature(Feature.MaterializedViews)) { + val error = FeatureNotAvailable(Feature.MaterializedViews) + logger.warn(s"⚠️ ${error.message}") + return Future.successful(Left(error)) + } + + // Check quota + licenseManager.quotas.maxMaterializedViews match { + case Some(max) => + mvCounter().flatMap { count => + if (count >= max) { + val error = QuotaExceeded("materialized_views", count, max) + logger.warn(s"⚠️ ${error.message}") + Future.successful(Left(error)) + } else { + execute(create).map(Right(_)) + } + } + + case None => + execute(create).map(Right(_)) + } + + case _ => + execute(request).map(Right(_)) + } + } +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/licensing/DqlLicenseChecker.scala b/core/src/main/scala/app/softnetwork/elastic/client/licensing/DqlLicenseChecker.scala new file mode 100644 index 00000000..b8f7920e --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/licensing/DqlLicenseChecker.scala @@ -0,0 +1,64 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.licensing + +import akka.actor.ActorSystem +import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.licensing._ +import app.softnetwork.elastic.sql.query._ +import org.slf4j.Logger + +import scala.concurrent.{ExecutionContext, Future} + +/** License checker for DQL statements. + */ +class DqlLicenseChecker( + licenseManager: LicenseManager, + logger: Logger +)(implicit system: ActorSystem) + extends LicenseChecker[DqlStatement, ElasticResult[QueryResult]] { + + implicit val ec: ExecutionContext = system.dispatcher + + override def check( + request: DqlStatement, + execute: DqlStatement => Future[ElasticResult[QueryResult]] + ): Future[Either[LicenseError, ElasticResult[QueryResult]]] = { + + val quota = licenseManager.quotas + + request match { + case single: SingleSearch => + single.limit match { + case Some(l) if quota.maxQueryResults.exists(_ < l.limit) => + val error = QuotaExceeded( + "query_results", + l.limit, + quota.maxQueryResults.get + ) + logger.warn(s"⚠️ ${error.message}") + Future.successful(Left(error)) + + case _ => + execute(single).map(Right(_)) + } + + case _ => + execute(request).map(Right(_)) + } + } +} diff --git a/licensing/build.sbt b/licensing/build.sbt new file mode 100644 index 00000000..59ed8497 --- /dev/null +++ b/licensing/build.sbt @@ -0,0 +1,4 @@ +organization := "app.softnetwork.elastic" + +name := "softclient4es-licensing" + diff --git a/licensing/src/main/scala/app/softnetwork/elastic/licensing/DefaultLicenseManager.scala b/licensing/src/main/scala/app/softnetwork/elastic/licensing/DefaultLicenseManager.scala new file mode 100644 index 00000000..348086ab --- /dev/null +++ b/licensing/src/main/scala/app/softnetwork/elastic/licensing/DefaultLicenseManager.scala @@ -0,0 +1,78 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.licensing + +class DefaultLicenseManager extends LicenseManager { + + private var currentLicense: LicenseKey = LicenseKey( + id = "community", + licenseType = LicenseType.Community, + features = Set( + Feature.MaterializedViews, + Feature.JdbcDriver + ), + expiresAt = None + ) + + override def validate(key: String): Either[LicenseError, LicenseKey] = { + key match { + case k if k.startsWith("PRO-") => + val license = LicenseKey( + id = k, + licenseType = LicenseType.Pro, + features = Set( + Feature.MaterializedViews, + Feature.JdbcDriver, + Feature.UnlimitedResults + ), + expiresAt = None + ) + currentLicense = license + Right(license) + + case k if k.startsWith("ENT-") => + val license = LicenseKey( + id = k, + licenseType = LicenseType.Enterprise, + features = Set( + Feature.MaterializedViews, + Feature.JdbcDriver, + Feature.OdbcDriver, + Feature.UnlimitedResults + ), + expiresAt = None + ) + currentLicense = license + Right(license) + + case _ => + Left(InvalidLicense("Invalid license key format")) + } + } + + override def hasFeature(feature: Feature): Boolean = { + currentLicense.features.contains(feature) + } + + override def quotas: Quota = currentLicense.licenseType match { + case LicenseType.Community => Quota.Community + case LicenseType.Pro => Quota.Pro + case LicenseType.Enterprise => Quota.Enterprise + } + + override def licenseType: LicenseType = currentLicense.licenseType +} diff --git a/licensing/src/main/scala/app/softnetwork/elastic/licensing/LicenseChecker.scala b/licensing/src/main/scala/app/softnetwork/elastic/licensing/LicenseChecker.scala new file mode 100644 index 00000000..44dd33a3 --- /dev/null +++ b/licensing/src/main/scala/app/softnetwork/elastic/licensing/LicenseChecker.scala @@ -0,0 +1,60 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.licensing + +import scala.concurrent.Future + +/** Generic abstraction for license checking. + * + * Type parameters: + * - REQUEST: The request type to check (e.g., SQL statement, query) + * - RESULT: The result type to return (e.g., query result, DDL result) + */ +trait LicenseChecker[REQUEST, RESULT] { + + /** Check license for a request. + * + * @param request + * The request to check + * @param execute + * The actual execution function (if license allows) + * @return + * Either a license error or the execution result + */ + def check( + request: REQUEST, + execute: REQUEST => Future[RESULT] + ): Future[Either[LicenseError, RESULT]] +} + +/** Helper for creating checkers + */ +object LicenseChecker { + + /** No-op checker (always allows) + */ + def noOp[REQUEST, RESULT]: LicenseChecker[REQUEST, RESULT] = + new LicenseChecker[REQUEST, RESULT] { + override def check( + request: REQUEST, + execute: REQUEST => Future[RESULT] + ): Future[Either[LicenseError, RESULT]] = { + import scala.concurrent.ExecutionContext.Implicits.global + execute(request).map(Right(_)) + } + } +} diff --git a/licensing/src/main/scala/app/softnetwork/elastic/licensing/package.scala b/licensing/src/main/scala/app/softnetwork/elastic/licensing/package.scala new file mode 100644 index 00000000..98175b28 --- /dev/null +++ b/licensing/src/main/scala/app/softnetwork/elastic/licensing/package.scala @@ -0,0 +1,116 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic + +package object licensing { + + sealed trait LicenseType + + object LicenseType { + case object Community extends LicenseType // Gratuit + case object Pro extends LicenseType // Payant + case object Enterprise extends LicenseType // Payant + support + } + + sealed trait Feature + + object Feature { + case object MaterializedViews extends Feature + case object JdbcDriver extends Feature + case object OdbcDriver extends Feature + case object UnlimitedResults extends Feature + case object AdvancedAggregations extends Feature + def values: Seq[Feature] = Seq( + MaterializedViews, + JdbcDriver, + OdbcDriver, + UnlimitedResults, + AdvancedAggregations + ) + } + + case class LicenseKey( + id: String, + licenseType: LicenseType, + features: Set[Feature], + expiresAt: Option[java.time.Instant], + metadata: Map[String, String] = Map.empty + ) + + case class Quota( + maxMaterializedViews: Option[Int], // None = unlimited + maxQueryResults: Option[Int], // None = unlimited + maxConcurrentQueries: Option[Int] + ) + + object Quota { + val Community: Quota = Quota( + maxMaterializedViews = Some(3), + maxQueryResults = Some(10000), + maxConcurrentQueries = Some(5) + ) + + val Pro: Quota = Quota( + maxMaterializedViews = Some(50), + maxQueryResults = Some(1000000), + maxConcurrentQueries = Some(50) + ) + + val Enterprise: Quota = Quota( + maxMaterializedViews = None, // Unlimited + maxQueryResults = None, + maxConcurrentQueries = None + ) + } + + trait LicenseManager { + + /** Validate license key */ + def validate(key: String): Either[LicenseError, LicenseKey] + + /** Check if feature is available */ + def hasFeature(feature: Feature): Boolean + + /** Get current quotas */ + def quotas: Quota + + /** Get license type */ + def licenseType: LicenseType + } + + sealed trait LicenseError { + def message: String + def statusCode: Int = 402 // Payment Required + } + + case class InvalidLicense(reason: String) extends LicenseError { + def message: String = s"Invalid license: $reason" + } + + case class ExpiredLicense(expiredAt: java.time.Instant) extends LicenseError { + def message: String = s"License expired at $expiredAt" + } + + case class FeatureNotAvailable(feature: Feature) extends LicenseError { + def message: String = s"Feature $feature requires a paid license" + } + + case class QuotaExceeded(quota: String, current: Int, max: Int) extends LicenseError { + def message: String = s"Quota exceeded: $quota ($current/$max)" + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index 9e8cfb9a..fdbab19c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -16,7 +16,14 @@ package app.softnetwork.elastic -import app.softnetwork.elastic.sql.{BooleanValue, ObjectValue, StringValue, StringValues, Value} +import app.softnetwork.elastic.sql.{ + BooleanValue, + ObjectValue, + ObjectValues, + StringValue, + StringValues, + Value +} import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.schema.{ Column, @@ -28,12 +35,14 @@ import app.softnetwork.elastic.sql.schema.{ Schema, ScriptProcessor, SetProcessor, - Table + Table, + TableType } import app.softnetwork.elastic.sql.serialization._ import app.softnetwork.elastic.sql.time.TimeUnit import com.fasterxml.jackson.databind.JsonNode +import java.security.MessageDigest import scala.jdk.CollectionConverters._ package object schema { @@ -179,7 +188,9 @@ package object schema { fields: List[IndexField] = Nil, primaryKey: List[String] = Nil, partitionBy: Option[IndexDatePartition] = None, - options: Map[String, Value[_]] = Map.empty + options: Map[String, Value[_]] = Map.empty, + materializedViews: Option[List[String]] = None, + tableType: TableType = TableType.Regular ) object IndexMappings { @@ -248,11 +259,36 @@ package object schema { case _ => None } + val materializedViews: Option[List[String]] = meta + .map { + case m: ObjectValue => + m.value.get("materialized_views") match { + case Some(mvs: StringValues) => mvs.values.map(_.ddl.replaceAll("\"", "")).toList + case Some(mv: StringValue) => List(mv.ddl.replaceAll("\"", "")) + case _ => List.empty + } + case _ => List.empty + } + .filter(_.nonEmpty) + + val tableType: TableType = meta + .flatMap { + case m: ObjectValue => + m.value.get("type") match { + case Some(mv: StringValue) => Some(TableType(mv.value)) + case _ => None + } + case _ => None + } + .getOrElse(TableType.Regular) + IndexMappings( fields = fields, primaryKey = primaryKey, partitionBy = partitionBy, - options = options + options = options, + materializedViews = materializedViews, + tableType = tableType ) } @@ -465,7 +501,9 @@ package object schema { mappings = esMappings.options, settings = esSettings.options, processors = processors.toSeq, - aliases = aliases + aliases = aliases, + materializedViews = esMappings.materializedViews.getOrElse(Nil), + tableType = esMappings.tableType ).update() } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index e01f33a2..cf2bfb74 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -389,6 +389,18 @@ object Parser TruncateTable(name) } + def createOrReplaceMaterializedView: PackratParser[CreateMaterializedView] = + ("CREATE" ~ "OR" ~ "REPLACE" ~ "MATERIALIZED" ~ "VIEW") ~ ident ~ ("AS" ~> dqlStatement) ^^ { + case _ ~ view ~ dql => + CreateMaterializedView(view, dql, ifNotExists = false, orReplace = true) + } + + def createMaterializedView: PackratParser[CreateMaterializedView] = + ("CREATE" ~ "MATERIALIZED" ~ "VIEW") ~ ifNotExists ~ ident ~ ("AS" ~> dqlStatement) ^^ { + case _ ~ ine ~ view ~ dql => + CreateMaterializedView(view, dql, ifNotExists = ine, orReplace = false) + } + def addColumn: PackratParser[AddColumn] = ("ADD" ~ "COLUMN") ~ ifNotExists ~ column ^^ { case _ ~ ine ~ col => AddColumn(col, ifNotExists = ine) @@ -588,7 +600,9 @@ object Parser dropPipeline | showPipeline | showCreatePipeline | - describePipeline + describePipeline | + createMaterializedView | + createOrReplaceMaterializedView def onConflict: PackratParser[OnConflict] = ("ON" ~ "CONFLICT" ~> opt(conflictTarget) <~ "DO") ~ ("UPDATE" | "NOTHING") ^^ { @@ -897,7 +911,7 @@ trait Parser }) >> cast private val regexAlias = - s"""\\b(?i)(?!(?:${reservedKeywords.mkString("|")})\\b)[a-zA-Z0-9_]*""".stripMargin + s"""\\b(?i)(?!(?:${reservedKeywords.mkString("|")})\\b)[a-zA-Z0-9_.]*""".stripMargin def alias: PackratParser[Alias] = Alias.regex.? ~ regexAlias.r ^^ { case _ ~ b => Alias(b) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index 4b18a3d4..ccebe4a7 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -27,10 +27,10 @@ import app.softnetwork.elastic.sql.schema.{ PartitionDate, RemoveProcessor, RenameProcessor, - Schema, ScriptProcessor, SetProcessor, - Table => DdlTable + Table => Schema, + TableType } import app.softnetwork.elastic.sql.function.aggregate.WindowFunction import app.softnetwork.elastic.sql.serialization._ @@ -561,6 +561,26 @@ package object query { sealed trait TableStatement extends DdlStatement + case class CreateMaterializedView( + view: String, + ddl: DqlStatement, + ifNotExists: Boolean = false, + orReplace: Boolean = false, + options: Map[String, Value[_]] = Map.empty + ) extends TableStatement { + override def sql: String = { + val replaceClause = if (orReplace) " OR REPLACE" else "" + val ineClause = if (!orReplace && ifNotExists) " IF NOT EXISTS" else "" + s"CREATE$replaceClause MATERIALIZED VIEW$ineClause $view AS ${ddl.sql}" + } + + lazy val search: SingleSearch = ddl match { + case s: SingleSearch => s + case _ => throw new IllegalArgumentException("Materialized view must be a single search") + } + + } + case class CreateTable( table: String, ddl: Either[DqlStatement, List[Column]], @@ -645,14 +665,39 @@ package object query { case None => Map.empty } - lazy val schema: Schema = DdlTable( + lazy val materializedViews: List[String] = options.get("materialized_views") match { + case Some(value) => + value match { + case ov: StringValues => + ov.values.flatMap { + case o: StringValue => + Some(o.value) + case _ => None + }.toList + case _ => Nil + } + case None => Nil + } + + lazy val tableType: TableType = (options.get("type") match { + case Some(value) => + value match { + case s: StringValue => Some(TableType(s.value)) + case _ => None + } + case None => None + }).getOrElse(TableType.Regular) + + lazy val schema: Schema = Schema( name = table, columns = columns.toList, primaryKey = primaryKey, partitionBy = partitionBy, mappings = mappings, settings = settings, - aliases = aliases + aliases = aliases, + materializedViews = materializedViews, + tableType = tableType ).update() lazy val defaultPipeline: IngestPipeline = schema.defaultPipeline diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 16512c40..75119f75 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -1833,6 +1833,42 @@ package object schema { } } + sealed trait TableType { + def name: String + } + + object TableType { + case object Regular extends TableType { + override def name: String = "regular" + } + case object External extends TableType { + override def name: String = "external" + } + case object Changelog extends TableType { + override def name: String = "changelog" + } + case object Enrichment extends TableType { + override def name: String = "enrichment" + } + case object View extends TableType { + override def name: String = "view" + } + case object MaterializedView extends TableType { + override def name: String = "materialized_view" + } + + def apply(name: String): TableType = + name.toLowerCase match { + case "regular" => Regular + case "external" => External + case "changelog" => Changelog + case "enrichment" => Enrichment + case "view" => View + case "materialized_view" => MaterializedView + case other => throw new Exception(s"Unknown table type: $other") + } + } + /** Definition of a table within the schema * * @param name @@ -1851,6 +1887,10 @@ package object schema { * optional list of ingest processors associated to this table (apart from column processors) * @param aliases * optional map of aliases associated to this table + * @param materializedViews + * optional list of materialized views associated to this table + * @param materializedView + * whether this table is a materialized view */ case class Table( name: String, @@ -1860,8 +1900,14 @@ package object schema { mappings: Map[String, Value[_]] = Map.empty, settings: Map[String, Value[_]] = Map.empty, processors: Seq[IngestProcessor] = Seq.empty, - aliases: Map[String, Value[_]] = Map.empty + aliases: Map[String, Value[_]] = Map.empty, + materializedViews: List[String] = Nil, + tableType: TableType = TableType.Regular ) extends DdlToken { + lazy val isRegular: Boolean = tableType == TableType.Regular + + lazy val isPartitioned: Boolean = partitionBy.isDefined + lazy val indexName: String = name.toLowerCase private[schema] lazy val cols: Map[String, Column] = columns.map(c => c.name -> c).toMap @@ -1891,7 +1937,13 @@ package object schema { ) ) .getOrElse(Map.empty) ++ - Map("columns" -> ObjectValue(cols.map { case (name, col) => name -> ObjectValue(col._meta) })) + Map( + "columns" -> ObjectValue(cols.map { case (name, col) => name -> ObjectValue(col._meta) }) + ) ++ Map( + "type" -> StringValue(tableType.name) + ) ++ Map( + "materialized_views" -> StringValues(materializedViews.map(StringValue)) + ) def update(): Table = { val updated = @@ -1901,7 +1953,7 @@ package object schema { "_meta" -> ObjectValue(updated.mappings.get("_meta") match { case Some(ObjectValue(value)) => - (value - "primary_key" - "partition_by" - "columns") ++ updated._meta + (value - "primary_key" - "partition_by" - "columns" - "materialized_views" - "type") ++ updated._meta case _ => updated._meta }) ) @@ -1949,6 +2001,8 @@ package object schema { .toSeq ++ implicitly[Seq[IngestProcessor]](primaryKey) def merge(statements: Seq[AlterTableStatement]): Table = { + if (!isRegular) + throw new Exception(s"Cannot alter table $name of type ${tableType.name}") statements .foldLeft(this) { (table, statement) => statement match { @@ -2341,7 +2395,7 @@ package object schema { val template = mapper.createObjectNode() template.set("mappings", indexMappings) template.set("settings", indexSettings) - if (aliases.nonEmpty) { + if (indexAliases.nonEmpty) { val aliasesNode = mapper.createObjectNode() indexAliases.foreach { alias => aliasesNode.set(alias.alias, alias.node) diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index 1178abda..817722ef 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -239,7 +239,9 @@ class ParserSpec extends AnyFlatSpec with Matchers { import Queries._ - "Parser" should "parse numerical EQ" in { + behavior of "Parser DQL" + + it should "parse numerical EQ" in { val result = Parser(numericalEq) result.toOption .map(_.sql) @@ -939,6 +941,8 @@ class ParserSpec extends AnyFlatSpec with Matchers { // --- DDL --- + behavior of "Parser DDL" + it should "parse CREATE TABLE if not exists" in { val sql = """CREATE TABLE IF NOT EXISTS users ( @@ -1498,6 +1502,8 @@ class ParserSpec extends AnyFlatSpec with Matchers { // --- DML --- + behavior of "Parser DML" + it should "parse INSERT INTO ... VALUES" in { val sql = "INSERT INTO users (id, name) VALUES (1, 'Alice') ON CONFLICT DO NOTHING" val result = Parser(sql) @@ -1537,6 +1543,32 @@ class ParserSpec extends AnyFlatSpec with Matchers { } } + it should "parse INSERT INTO ... SELECT with alias" in { + val sql = """INSERT INTO orders_with_customers_mv_customers_changelog AS + |SELECT + | id, + | name, + | email, + | department.zipcode AS department.zip_code + |FROM customers; + |""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case Insert( + "orders_with_customers_mv_customers_changelog", + Nil, + Left(sel: DqlStatement), + None + ) => + sel.sql should include( + "SELECT id, name, email, department.zipcode AS department.zip_code FROM customers" + ) + case _ => fail("Expected Insert with select") + } + } + it should "parse INSERT INTO ... SELECT with ON CONFLICT" in { val sql = "INSERT INTO users SELECT id, name FROM old_users ON CONFLICT (id) DO UPDATE" val result = Parser(sql) @@ -1589,4 +1621,5 @@ class ParserSpec extends AnyFlatSpec with Matchers { case _ => fail("Expected Union") } } + } From e8a9b67b6a25dce1813556c2be5707180254222a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 16 Jan 2026 18:18:07 +0100 Subject: [PATCH 15/22] fix tranformCheckNotNull --- sql/src/main/scala/app/softnetwork/elastic/sql/package.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 1dcda092..ded20f04 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -1011,10 +1011,10 @@ package object sql { else "" lazy val transformCheckNotNull: Option[String] = - if (path.isEmpty || !nullable) None + if (aliasOrName.isEmpty || !nullable) None else Option( - s"(doc['$transformParamName'].size() == 0 ? $nullValue : doc['$transformParamName'].value${painlessMethods + s"(doc['$aliasOrName'].size() == 0 ? $nullValue : doc['$aliasOrName'].value${painlessMethods .mkString("")})" ) From 18009570b2058f1cd2c53c5551abd44eb8dd7444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 16 Jan 2026 18:18:58 +0100 Subject: [PATCH 16/22] add schema lineage --- .../softnetwork/elastic/schema/package.scala | 49 +++++++++++++++++-- .../elastic/sql/schema/package.scala | 37 +++++++++++++- 2 files changed, 81 insertions(+), 5 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index fdbab19c..2ec3ade3 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -55,7 +55,8 @@ package object schema { not_null: Option[Boolean] = None, comment: Option[String] = None, fields: List[IndexField] = Nil, - options: Map[String, Value[_]] = Map.empty + options: Map[String, Value[_]] = Map.empty, + lineage: Map[String, Seq[(String, String)]] = Map.empty // ✅ Added ) { lazy val ddlColumn: Column = { Column( @@ -66,7 +67,8 @@ package object schema { defaultValue = null_value, notNull = not_null.getOrElse(false), comment = comment, - options = options + options = options, + lineage = lineage // ✅ Added ) } } @@ -170,6 +172,46 @@ package object schema { } case _ => None } + + // ✅ Extract lineage from _meta + val lineage = _meta + .flatMap { + case m: ObjectValue => + m.value.get("lineage") match { + case Some(lin: ObjectValue) => + Some( + lin.value.flatMap { + case (pathId, pathValue: ObjectValues) => + // Parse the chain of (table, column) pairs + val chain = pathValue.values.flatMap { + case obj: ObjectValue => + val tableOpt = obj.value.get("table") match { + case Some(StringValue(t)) => Some(t) + case _ => None + } + val columnOpt = obj.value.get("column") match { + case Some(StringValue(c)) => Some(c) + case _ => None + } + for { + table <- tableOpt + column <- columnOpt + } yield (table, column) + case _ => None + } + + if (chain.nonEmpty) Some(pathId -> chain) + else None + + case _ => None + } + ) + case _ => None + } + case _ => None + } + .getOrElse(Map.empty) + IndexField( name = name, `type` = tpe, @@ -178,7 +220,8 @@ package object schema { not_null = notNull, comment = comment, fields = fields, - options = options + options = options, + lineage = lineage ) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 75119f75..e905af66 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -1391,6 +1391,8 @@ package object schema { * optional options for this column (search analyzer, ...) * @param struct * optional parent struct column + * @param lineage + * sequence of (table, column) pairs indicating the lineage of this column */ case class Column( name: String, @@ -1401,7 +1403,8 @@ package object schema { notNull: Boolean = false, comment: Option[String] = None, options: Map[String, Value[_]] = Map.empty, - struct: Option[Column] = None + struct: Option[Column] = None, + lineage: Map[String, Seq[(String, String)]] = Map.empty // ✅ Key = path ID, Value = chain ) extends DdlToken { def path: String = struct.map(st => s"${st.name}.$name").getOrElse(name) private def level: Int = struct.map(_.level + 1).getOrElse(0) @@ -1418,6 +1421,18 @@ package object schema { } } + /** Flattens all lineage paths into a set of (table, column) pairs */ + def allLineageSources: Set[(String, String)] = + lineage.values.flatten.toSet + + /** Gets the immediate sources (last element of each path) */ + def immediateSources: Set[(String, String)] = + lineage.values.flatMap(_.lastOption).toSet + + /** Gets the original sources (first element of each path) */ + def originalSources: Set[(String, String)] = + lineage.values.flatMap(_.headOption).toSet + def _meta: Map[String, Value[_]] = { Map( "data_type" -> StringValue(dataType.typeId), @@ -1438,7 +1453,25 @@ package object schema { "multi_fields" -> ObjectValue( multiFields.map(field => field.name -> ObjectValue(field._meta)).toMap ) - ) + ) ++ (if (lineage.nonEmpty) { + // ✅ Lineage as map of paths + Map( + "lineage" -> ObjectValue( + lineage.map { case (pathId, chain) => + pathId -> ObjectValues( + chain.map { case (table, column) => + ObjectValue( + Map( + "table" -> StringValue(table), + "column" -> StringValue(column) + ) + ) + } + ) + } + ) + ) + } else Map.empty) } def updateStruct(): Column = { From 0c492be2d7445261fae96bd703f2b280017459d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 16 Jan 2026 18:19:38 +0100 Subject: [PATCH 17/22] implements implicit Criteria to JsonNode --- .../elastic/sql/bridge/package.scala | 37 ++++++++++++++++ .../elastic/sql/bridge/package.scala | 42 +++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 9da37706..36c73605 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -28,8 +28,11 @@ import app.softnetwork.elastic.sql.function.aggregate.COUNT import app.softnetwork.elastic.sql.function.geo.{Distance, Meters} import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql.query._ +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.NullNode import com.sksamuel.elastic4s.ElasticApi import com.sksamuel.elastic4s.ElasticApi._ +import com.sksamuel.elastic4s.json.JacksonBuilder import com.sksamuel.elastic4s.requests.script.Script import com.sksamuel.elastic4s.requests.script.ScriptType.Source import com.sksamuel.elastic4s.requests.searches.aggs.{ @@ -1042,6 +1045,40 @@ package object bridge { ) } + implicit def queryToJson( + query: Query + ): JsonNode = { + JacksonBuilder.toNode( + SearchBodyBuilderFn( + ElasticApi.search("") query { + query + } + ).value + ) match { + case Left(node: JsonNode) => + if (node.has("query")) { + node.get("query") + } else { + node + } + case Right(_) => NullNode.instance + } + } + + implicit def criteriaToQuery(criteria: Criteria)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Query = { + ElasticCriteria(criteria).asQuery() + } + + implicit def criteriaToNode(criteria: Criteria)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): JsonNode = { + queryToJson(criteriaToQuery(criteria)) + } + implicit def filterToQuery( filter: ElasticFilter ): ElasticBridge = { diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 2f22881a..1a1b2ad6 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -28,6 +28,9 @@ import app.softnetwork.elastic.sql.function.aggregate.COUNT import app.softnetwork.elastic.sql.function.geo.{Distance, Meters} import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql.query._ +import app.softnetwork.elastic.sql.schema.mapper +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.NullNode import com.sksamuel.elastic4s.{ElasticApi, FetchSourceContext} import com.sksamuel.elastic4s.ElasticApi._ import com.sksamuel.elastic4s.http.ElasticDsl.BuildableTermsNoOp @@ -45,6 +48,7 @@ import com.sksamuel.elastic4s.searches.{MultiSearchRequest, SearchRequest} import com.sksamuel.elastic4s.searches.sort.{FieldSort, ScriptSort, ScriptSortType} import scala.language.implicitConversions +import scala.util.{Failure, Success, Try} package object bridge { @@ -1041,6 +1045,44 @@ package object bridge { ) } + implicit def queryToString( + query: Query + ): String = { + SearchBodyBuilderFn( + ElasticApi.search("") query { + query + } + ).string() + } + + implicit def queryToJson( + query: Query + ): JsonNode = { + Try(mapper.readTree(queryToString(query))) match { + case Success(node) => + if (node.has("query")) { + node.get("query") + } else { + node + } + case Failure(_) => NullNode.instance + } + } + + implicit def criteriaToQuery(criteria: Criteria)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Query = { + ElasticCriteria(criteria).asQuery() + } + + implicit def criteriaToNode(criteria: Criteria)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): JsonNode = { + queryToJson(criteriaToQuery(criteria)) + } + implicit def filterToQuery( filter: ElasticFilter ): ElasticBridge = { From e229b5e307db35ab574a05a984f4e9557fd87305 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sat, 17 Jan 2026 07:34:33 +0100 Subject: [PATCH 18/22] fix parser specifications --- .../app/softnetwork/elastic/sql/parser/ParserSpec.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index 817722ef..de590577 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -998,16 +998,16 @@ class ParserSpec extends AnyFlatSpec with Matchers { println(schema.defaultPipeline.ddl) val json = schema.defaultPipeline.json println(json) - json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET VALUE 'anonymous' IF ctx.name == null, age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at SET VALUE _ingest.timestamp IF ctx.ingested_at == null, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name SET VALUE 'anonymous' IF ctx.name == null","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at SET VALUE _ingest.timestamp IF ctx.ingested_at == null","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3","ignore_failure":true}},{"date_index_name":{"field":"birthdate","description":"PARTITION BY birthdate (MONTH)","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at SET DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name SET DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at SET DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3","ignore_failure":true}},{"date_index_name":{"field":"birthdate","description":"PARTITION BY birthdate (MONTH)","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" val indexMappings = schema.indexMappings println(indexMappings) - indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer"},"name":{"type":"text","fields":{"raw":{"type":"keyword"}},"analyzer":"french","search_analyzer":"french"},"birthdate":{"type":"date"},"age":{"type":"integer"},"ingested_at":{"type":"date"},"profile":{"type":"object","properties":{"bio":{"type":"text"},"followers":{"type":"integer"},"join_date":{"type":"date"},"seniority":{"type":"integer"}}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"},"columns":{"age":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(birthdate, CURRENT_DATE, YEAR)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3"}},"profile":{"data_type":"STRUCT","not_null":"false","comment":"user profile","multi_fields":{"bio":{"data_type":"VARCHAR","not_null":"false"},"followers":{"data_type":"INT","not_null":"false"},"join_date":{"data_type":"DATE","not_null":"false"},"seniority":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3"}}}},"name":{"data_type":"VARCHAR","not_null":"false","default_value":"anonymous","multi_fields":{"raw":{"data_type":"KEYWORD","not_null":"false","comment":"sortable"}}},"ingested_at":{"data_type":"TIMESTAMP","not_null":"false","default_value":"_ingest.timestamp"},"birthdate":{"data_type":"DATE","not_null":"false"},"id":{"data_type":"INT","not_null":"true","comment":"user identifier"}}}}""".stripMargin + indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer"},"name":{"type":"text","fields":{"raw":{"type":"keyword"}},"analyzer":"french","search_analyzer":"french"},"birthdate":{"type":"date"},"age":{"type":"integer"},"ingested_at":{"type":"date"},"profile":{"type":"object","properties":{"bio":{"type":"text"},"followers":{"type":"integer"},"join_date":{"type":"date"},"seniority":{"type":"integer"}}}},"dynamic":false,"_meta":{"columns":{"age":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(birthdate, CURRENT_DATE, YEAR)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3"}},"profile":{"data_type":"STRUCT","not_null":"false","comment":"user profile","multi_fields":{"bio":{"data_type":"VARCHAR","not_null":"false"},"followers":{"data_type":"INT","not_null":"false"},"join_date":{"data_type":"DATE","not_null":"false"},"seniority":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3"}}}},"name":{"data_type":"VARCHAR","not_null":"false","default_value":"anonymous","multi_fields":{"raw":{"data_type":"KEYWORD","not_null":"false","comment":"sortable"}}},"ingested_at":{"data_type":"TIMESTAMP","not_null":"false","default_value":"_ingest.timestamp"},"birthdate":{"data_type":"DATE","not_null":"false"},"id":{"data_type":"INT","not_null":"true","comment":"user identifier"}},"materialized_views":[],"primary_key":["id"],"type":"regular","partition_by":{"column":"birthdate","granularity":"M"}}}""".stripMargin val indexSettings = schema.indexSettings println(indexSettings) indexSettings.toString shouldBe """{"index":{}}""" val pipeline = schema.defaultPipelineNode println(pipeline) - pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET VALUE 'anonymous' IF ctx.name == null, age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at SET VALUE _ingest.timestamp IF ctx.ingested_at == null, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name SET VALUE 'anonymous' IF ctx.name == null","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at SET VALUE _ingest.timestamp IF ctx.ingested_at == null","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3","ignore_failure":true}},{"date_index_name":{"field":"birthdate","description":"PARTITION BY birthdate (MONTH)","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at SET DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name SET DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at SET DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3","ignore_failure":true}},{"date_index_name":{"field":"birthdate","description":"PARTITION BY birthdate (MONTH)","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" // Reconstruct EsIndex val mappings = mapper.createObjectNode() mappings.set("mappings", indexMappings) From 9ff1b6c117c6202a4e9ef2f10b68d838978e4e2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sat, 17 Jan 2026 09:09:36 +0100 Subject: [PATCH 19/22] implements extension registry --- ...pp.softnetwork.elastic.client.ExtensionSpi | 2 + .../elastic/client/ElasticClientApi.scala | 1 + .../elastic/client/ExtensionApi.scala | 29 ++++ .../elastic/client/ExtensionRegistry.scala | 80 ++++++++++ .../elastic/client/ExtensionSpi.scala | 93 ++++++++++++ .../elastic/client/GatewayApi.scala | 101 ++++--------- .../client/extensions/CoreDdlExtension.scala | 106 +++++++++++++ .../client/extensions/CoreDqlExtension.scala | 139 ++++++++++++++++++ .../client/licensing/DdlLicenseChecker.scala | 73 --------- .../client/licensing/DqlLicenseChecker.scala | 64 -------- .../elastic/sql/query/package.scala | 4 +- 11 files changed, 479 insertions(+), 213 deletions(-) create mode 100644 core/src/main/resources/META-INF/services/app.softnetwork.elastic.client.ExtensionSpi create mode 100644 core/src/main/scala/app/softnetwork/elastic/client/ExtensionApi.scala create mode 100644 core/src/main/scala/app/softnetwork/elastic/client/ExtensionRegistry.scala create mode 100644 core/src/main/scala/app/softnetwork/elastic/client/ExtensionSpi.scala create mode 100644 core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDdlExtension.scala create mode 100644 core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtension.scala delete mode 100644 core/src/main/scala/app/softnetwork/elastic/client/licensing/DdlLicenseChecker.scala delete mode 100644 core/src/main/scala/app/softnetwork/elastic/client/licensing/DqlLicenseChecker.scala diff --git a/core/src/main/resources/META-INF/services/app.softnetwork.elastic.client.ExtensionSpi b/core/src/main/resources/META-INF/services/app.softnetwork.elastic.client.ExtensionSpi new file mode 100644 index 00000000..d2e812cf --- /dev/null +++ b/core/src/main/resources/META-INF/services/app.softnetwork.elastic.client.ExtensionSpi @@ -0,0 +1,2 @@ +app.softnetwork.elastic.client.extensions.CoreDdlExtension +app.softnetwork.elastic.client.extensions.CoreDqlExtension \ No newline at end of file diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala index 3cf7e22b..9e2f8723 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala @@ -48,6 +48,7 @@ trait ElasticClientApi with SerializationApi with PipelineApi with TemplateApi + with ExtensionApi with GatewayApi with ClientCompanion { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ExtensionApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ExtensionApi.scala new file mode 100644 index 00000000..80d9eda8 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/ExtensionApi.scala @@ -0,0 +1,29 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.licensing.{DefaultLicenseManager, LicenseManager} + +trait ExtensionApi { _: ElasticClientApi => + + // ✅ Inject license manager (overridable) + def licenseManager: LicenseManager = new DefaultLicenseManager() + + /** Extension registry (lazy loaded) */ + lazy val extensionRegistry: ExtensionRegistry = + new ExtensionRegistry(config, licenseManager) +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ExtensionRegistry.scala b/core/src/main/scala/app/softnetwork/elastic/client/ExtensionRegistry.scala new file mode 100644 index 00000000..d10c17ea --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/ExtensionRegistry.scala @@ -0,0 +1,80 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.licensing.LicenseManager +import app.softnetwork.elastic.sql.query.Statement +import com.typesafe.config.Config + +/** Registry for managing loaded extensions. + */ +class ExtensionRegistry( + config: Config, + licenseManager: LicenseManager +) { + + import scala.jdk.CollectionConverters._ + import java.util.ServiceLoader + + private val logger = org.slf4j.LoggerFactory.getLogger(getClass) + + /** All loaded extensions. + */ + lazy val extensions: Seq[ExtensionSpi] = { + val loader = ServiceLoader.load(classOf[ExtensionSpi]) + + val loaded = loader.iterator().asScala.toSeq.flatMap { ext => + logger.info( + s"🔌 Discovered extension: ${ext.extensionName} v${ext.version} (priority: ${ext.priority})" + ) + + ext.initialize(config, licenseManager) match { + case Right(_) => + logger.info(s"✅ Extension ${ext.extensionName} initialized successfully") + Some(ext) + + case Left(error) => + logger.warn(s"⚠️ Failed to initialize extension ${ext.extensionName}: $error") + None + } + } + + // ✅ Sort by priority (lower = higher priority) + loaded.sortBy(_.priority) + } + + /** Find extension that can handle a statement. + */ + def findHandler(statement: Statement): Option[ExtensionSpi] = { + extensions.find(_.canHandle(statement)) + } + + /** Get all supported syntax. + */ + def allSupportedSyntax: Seq[String] = { + extensions.flatMap(_.supportedSyntax).distinct + } + + /** Shutdown all extensions. + */ + def shutdown(): Unit = { + extensions.foreach { ext => + logger.info(s"🔌 Shutting down extension: ${ext.extensionName}") + ext.shutdown() + } + } +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ExtensionSpi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ExtensionSpi.scala new file mode 100644 index 00000000..52576194 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/ExtensionSpi.scala @@ -0,0 +1,93 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.actor.ActorSystem +import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.licensing.LicenseManager +import app.softnetwork.elastic.sql.query.Statement +import com.typesafe.config.Config + +import scala.concurrent.Future + +/** Service Provider Interface for Elasticsearch extensions. + * + * Extensions can add new capabilities to the client (e.g., materialized views, advanced analytics, + * custom drivers) while keeping the core API stable. + */ +trait ExtensionSpi { + + /** Unique identifier for this extension. + */ + def extensionId: String + + /** Human-readable name. + */ + def extensionName: String + + /** Version of the extension. + */ + def version: String + + /** Priority for extension resolution (lower = higher priority). + * + * Default: 100 (core extensions) Premium extensions should use lower values (e.g., 50, 10) + */ + def priority: Int = 100 + + /** Initialize the extension with configuration and license manager. + * + * @param config + * Extension-specific configuration + * @param licenseManager + * License manager for checking quotas + * @return + * Success or error message + */ + def initialize( + config: Config, + licenseManager: LicenseManager + ): Either[String, Unit] + + /** Check if this extension can handle a given SQL statement. + */ + def canHandle(statement: Statement): Boolean + + /** Execute a statement handled by this extension. + * + * @param statement + * SQL statement to execute + * @param client + * Elasticsearch client API for low-level operations + * @return + * Query result + */ + def execute( + statement: Statement, + client: ElasticClientApi + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] + + /** List of SQL keywords/syntax added by this extension. + * + * Example: ["CREATE MATERIALIZED VIEW", "REFRESH MATERIALIZED VIEW"] + */ + def supportedSyntax: Seq[String] + + /** Shutdown hook for cleanup. + */ + def shutdown(): Unit = () +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala index 70652228..48e7a2d9 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala @@ -17,7 +17,6 @@ package app.softnetwork.elastic.client import akka.actor.ActorSystem -import app.softnetwork.elastic.client.licensing.{DdlLicenseChecker, DqlLicenseChecker} import app.softnetwork.elastic.client.result.{ DdlResult, DmlResult, @@ -33,7 +32,6 @@ import app.softnetwork.elastic.client.result.{ SQLResult, TableResult } -import app.softnetwork.elastic.licensing.{DefaultLicenseManager, LicenseChecker, LicenseManager} import app.softnetwork.elastic.sql.parser.Parser import app.softnetwork.elastic.sql.query.{ AlterTable, @@ -1157,33 +1155,7 @@ class DdlRouterExecutor( } trait GatewayApi extends ElasticClientHelpers { - _: IndicesApi - with PipelineApi - with MappingApi - with SettingsApi - with AliasApi - with TemplateApi - with SearchApi - with ScrollApi - with VersionApi => - - // ✅ Inject license manager (overridable) - def licenseManager: LicenseManager = new DefaultLicenseManager() - - // ✅ Create checkers (depend on licenseManager) - def dqlChecker(implicit - system: ActorSystem - ): LicenseChecker[DqlStatement, ElasticResult[QueryResult]] = - new DqlLicenseChecker(licenseManager, logger) - - def ddlChecker(implicit - system: ActorSystem - ): LicenseChecker[DdlStatement, ElasticResult[QueryResult]] = - new DdlLicenseChecker( - licenseManager, - logger, - () => countMaterializedViews() - ) + self: ElasticClientApi => lazy val dqlExecutor = new DqlExecutor( api = this, @@ -1210,13 +1182,6 @@ trait GatewayApi extends ElasticClientHelpers { tableExec = tableExecutor ) - // ✅ Helper to count materialized views - private[client] def countMaterializedViews()(implicit system: ActorSystem): Future[Int] = { - implicit val ec: ExecutionContext = system.dispatcher - // Query Elasticsearch for indices with _meta.type = "materialized_view" - Future.successful(0) // TODO: Implement - } - // ======================================================================== // SQL GATEWAY API // ======================================================================== @@ -1284,50 +1249,36 @@ trait GatewayApi extends ElasticClientHelpers { )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { implicit val ec: ExecutionContext = system.dispatcher - statement match { + // ✅ TRY EXTENSIONS FIRST + extensionRegistry.findHandler(statement) match { + case Some(extension) => + logger.info(s"🔌 Routing to extension: ${extension.extensionName}") + extension.execute(statement, self) // ✅ Pass full client API - // ✅ DQL with license check - case dql: DqlStatement => - dqlChecker.check(dql, dqlExecutor.execute).map { - case Right(result) => result - case Left(licenseError) => - ElasticFailure( - ElasticError( - message = licenseError.message, - statusCode = Some(licenseError.statusCode), - operation = Some("license") - ) - ) - } + case None => + // ✅ FALLBACK TO STANDARD EXECUTORS + statement match { + case dql: DqlStatement => + logger.debug("🔧 Executing DQL with base executor") + dqlExecutor.execute(dql) - // ✅ DML (no license check for now) - case dml: DmlStatement => - dmlExecutor.execute(dml) + case dml: DmlStatement => + logger.debug("🔧 Executing DML with base executor") + dmlExecutor.execute(dml) - // ✅ DDL with license check - case ddl: DdlStatement => - ddlChecker.check(ddl, ddlExecutor.execute).map { - case Right(result) => result - case Left(licenseError) => - ElasticFailure( - ElasticError( - message = licenseError.message, - statusCode = Some(licenseError.statusCode), - operation = Some("license") - ) + case ddl: DdlStatement => + logger.debug("🔧 Executing DDL with base executor") + ddlExecutor.execute(ddl) + + case _ => + val error = ElasticError( + message = s"Unsupported SQL statement: $statement", + statusCode = Some(400), + operation = Some("schema") ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) } - - case _ => - // unsupported SQL statement - val error = - ElasticError( - message = s"Unsupported SQL statement: $statement", - statusCode = Some(400), - operation = Some("schema") - ) - logger.error(s"❌ ${error.message}") - Future.successful(ElasticFailure(error)) } } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDdlExtension.scala b/core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDdlExtension.scala new file mode 100644 index 00000000..33a59092 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDdlExtension.scala @@ -0,0 +1,106 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.extensions + +import akka.actor.ActorSystem +import app.softnetwork.elastic.client._ +import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.licensing._ +import app.softnetwork.elastic.sql.query._ +import com.typesafe.config.Config + +import scala.concurrent.{ExecutionContext, Future} + +//format:off +/** Core extension for basic DDL operations. + * + * {{{ + * ✅ OSS (always loaded) + * ✅ Handles simple DDL (CREATE TABLE, DROP TABLE, etc.) + * ✅ Does NOT handle materialized views (premium extension) + * }}} + */ +//format:on +class CoreDdlExtension extends ExtensionSpi { + + private var licenseManager: Option[LicenseManager] = None + private val logger = org.slf4j.LoggerFactory.getLogger(getClass) + + override def extensionId: String = "core-ddl" + override def extensionName: String = "Core DDL" + override def version: String = "0.1.0" + + override def initialize( + config: Config, + manager: LicenseManager + ): Either[String, Unit] = { + logger.info("🔌 Initializing Core DDL extension") + licenseManager = Some(manager) + Right(()) + } + + override def priority: Int = 100 + + override def canHandle(statement: Statement): Boolean = statement match { + case ddl: DdlStatement => + // ✅ Handle basic DDL, but NOT materialized views + ddl match { + case _: MaterializedViewStatement => false + case _ => true + } + case _ => false + } + + override def execute( + statement: Statement, + client: ElasticClientApi + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + + statement match { + case ddl: DdlStatement => + // ✅ No quota checks for basic DDL + client.ddlExecutor.execute(ddl) + + case _ => + Future.successful( + ElasticFailure( + ElasticError( + message = "Statement not supported by this extension", + statusCode = Some(400), + operation = Some("extension") + ) + ) + ) + } + } + + override def supportedSyntax: Seq[String] = Seq( + "CREATE TABLE", + "DROP TABLE", + "ALTER TABLE", + "TRUNCATE TABLE", + "SHOW CREATE TABLE", + "SHOW TABLE", + "DESCRIBE TABLE", + "CREATE PIPELINE", + "ALTER PIPELINE", + "DROP PIPELINE", + "SHOW PIPELINE", + "DESCRIBE PIPELINE" + ) +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtension.scala b/core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtension.scala new file mode 100644 index 00000000..ff4b0b4a --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtension.scala @@ -0,0 +1,139 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.extensions + +import akka.actor.ActorSystem +import app.softnetwork.elastic.client._ +import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.licensing._ +import app.softnetwork.elastic.sql.query._ +import com.typesafe.config.Config + +import scala.concurrent.{ExecutionContext, Future} + +//format:off +/** Core extension for DQL quota enforcement. + * + * {{{ + * ✅ OSS (always loaded) + * ✅ Applies Community quotas by default + * ✅ Can be upgraded with Pro license + * }}} + */ +//format:on +class CoreDqlExtension extends ExtensionSpi { + + private var licenseManager: Option[LicenseManager] = None + private val logger = org.slf4j.LoggerFactory.getLogger(getClass) + + override def extensionId: String = "core-dql" + override def extensionName: String = "Core DQL Quotas" + override def version: String = "0.1.0" + + override def initialize( + config: Config, + manager: LicenseManager + ): Either[String, Unit] = { + logger.info("🔌 Initializing Core DQL extension") + licenseManager = Some(manager) + Right(()) + } + + // ✅ Priority: 100 (lower = higher priority) + // Built-in extensions should have lower priority than premium ones + override def priority: Int = 100 + + override def canHandle(statement: Statement): Boolean = statement match { + case _: DqlStatement => true + case _ => false + } + + override def execute( + statement: Statement, + client: ElasticClientApi + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + + statement match { + case dql: DqlStatement => + licenseManager match { + case Some(manager) => + checkQuotasAndExecute(dql, manager, client) + case None => + // No license manager, execute without checks + client.dqlExecutor.execute(dql) + } + + case _ => + Future.successful( + ElasticFailure( + ElasticError( + message = "Statement not supported by this extension", + statusCode = Some(400), + operation = Some("extension") + ) + ) + ) + } + } + + override def supportedSyntax: Seq[String] = Seq( + "SELECT ... FROM ... WHERE ... GROUP BY ... HAVING ... LIMIT ..." + ) + + // ════════════════════════════════════════════════════════════════════ + // ✅ QUOTA CHECK LOGIC (in extension, not in core) + // ════════════════════════════════════════════════════════════════════ + + private def checkQuotasAndExecute( + dql: DqlStatement, + manager: LicenseManager, + client: ElasticClientApi + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + + val quota = manager.quotas + + dql match { + case single: SingleSearch => + single.limit match { + case Some(l) if quota.maxQueryResults.exists(_ < l.limit) => + logger.warn( + s"⚠️ Query result limit (${l.limit}) exceeds license quota (${quota.maxQueryResults.get})" + ) + Future.successful( + ElasticFailure( + ElasticError( + message = + s"Query result limit (${l.limit}) exceeds license quota (${quota.maxQueryResults.get}). Upgrade to Pro license.", + statusCode = Some(402), + operation = Some("license") + ) + ) + ) + + case _ => + // Quota OK, execute + client.dqlExecutor.execute(single) + } + + case _ => + // Other DQL types (no quota check needed) + client.dqlExecutor.execute(dql) + } + } +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/licensing/DdlLicenseChecker.scala b/core/src/main/scala/app/softnetwork/elastic/client/licensing/DdlLicenseChecker.scala deleted file mode 100644 index b9e745d8..00000000 --- a/core/src/main/scala/app/softnetwork/elastic/client/licensing/DdlLicenseChecker.scala +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2025 SOFTNETWORK - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package app.softnetwork.elastic.client.licensing - -import akka.actor.ActorSystem -import app.softnetwork.elastic.client.result._ -import app.softnetwork.elastic.licensing._ -import app.softnetwork.elastic.sql.query._ -import org.slf4j.Logger - -import scala.concurrent.{ExecutionContext, Future} - -/** License checker for DDL statements. - */ -class DdlLicenseChecker( - licenseManager: LicenseManager, - logger: Logger, - mvCounter: () => Future[Int] // ✅ Dependency injection -)(implicit system: ActorSystem) - extends LicenseChecker[DdlStatement, ElasticResult[QueryResult]] { - - implicit val ec: ExecutionContext = system.dispatcher - - override def check( - request: DdlStatement, - execute: DdlStatement => Future[ElasticResult[QueryResult]] - ): Future[Either[LicenseError, ElasticResult[QueryResult]]] = { - - request match { - case create: CreateMaterializedView => - // Check feature availability - if (!licenseManager.hasFeature(Feature.MaterializedViews)) { - val error = FeatureNotAvailable(Feature.MaterializedViews) - logger.warn(s"⚠️ ${error.message}") - return Future.successful(Left(error)) - } - - // Check quota - licenseManager.quotas.maxMaterializedViews match { - case Some(max) => - mvCounter().flatMap { count => - if (count >= max) { - val error = QuotaExceeded("materialized_views", count, max) - logger.warn(s"⚠️ ${error.message}") - Future.successful(Left(error)) - } else { - execute(create).map(Right(_)) - } - } - - case None => - execute(create).map(Right(_)) - } - - case _ => - execute(request).map(Right(_)) - } - } -} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/licensing/DqlLicenseChecker.scala b/core/src/main/scala/app/softnetwork/elastic/client/licensing/DqlLicenseChecker.scala deleted file mode 100644 index b8f7920e..00000000 --- a/core/src/main/scala/app/softnetwork/elastic/client/licensing/DqlLicenseChecker.scala +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2025 SOFTNETWORK - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package app.softnetwork.elastic.client.licensing - -import akka.actor.ActorSystem -import app.softnetwork.elastic.client.result._ -import app.softnetwork.elastic.licensing._ -import app.softnetwork.elastic.sql.query._ -import org.slf4j.Logger - -import scala.concurrent.{ExecutionContext, Future} - -/** License checker for DQL statements. - */ -class DqlLicenseChecker( - licenseManager: LicenseManager, - logger: Logger -)(implicit system: ActorSystem) - extends LicenseChecker[DqlStatement, ElasticResult[QueryResult]] { - - implicit val ec: ExecutionContext = system.dispatcher - - override def check( - request: DqlStatement, - execute: DqlStatement => Future[ElasticResult[QueryResult]] - ): Future[Either[LicenseError, ElasticResult[QueryResult]]] = { - - val quota = licenseManager.quotas - - request match { - case single: SingleSearch => - single.limit match { - case Some(l) if quota.maxQueryResults.exists(_ < l.limit) => - val error = QuotaExceeded( - "query_results", - l.limit, - quota.maxQueryResults.get - ) - logger.warn(s"⚠️ ${error.message}") - Future.successful(Left(error)) - - case _ => - execute(single).map(Right(_)) - } - - case _ => - execute(request).map(Right(_)) - } - } -} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index ccebe4a7..e71a4375 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -561,13 +561,15 @@ package object query { sealed trait TableStatement extends DdlStatement + sealed trait MaterializedViewStatement extends TableStatement + case class CreateMaterializedView( view: String, ddl: DqlStatement, ifNotExists: Boolean = false, orReplace: Boolean = false, options: Map[String, Value[_]] = Map.empty - ) extends TableStatement { + ) extends MaterializedViewStatement { override def sql: String = { val replaceClause = if (orReplace) " OR REPLACE" else "" val ineClause = if (!orReplace && ifNotExists) " IF NOT EXISTS" else "" From a502899094a4e94c875033a24e498955a71604f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sat, 17 Jan 2026 09:11:34 +0100 Subject: [PATCH 20/22] update version --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 8ad82d69..54b5effc 100644 --- a/build.sbt +++ b/build.sbt @@ -19,7 +19,7 @@ ThisBuild / organization := "app.softnetwork" name := "softclient4es" -ThisBuild / version := "0.15.0" +ThisBuild / version := "0.16.0" ThisBuild / scalaVersion := scala213 From 763c955d94f22a75cbb515c496bd0e18622bb683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Thu, 22 Jan 2026 19:33:36 +0100 Subject: [PATCH 21/22] update version, add enrich policy and transform apis, add support for materialized view ddl, add support for show tables ddl statement --- build.sbt | 5 +- .../elastic/client/ElasticClientApi.scala | 2 + .../client/ElasticClientDelegator.scala | 89 +++- .../elastic/client/ElasticsearchVersion.scala | 18 + .../elastic/client/EnrichPolicyApi.scala | 90 ++++ .../elastic/client/ExtensionApi.scala | 5 +- .../elastic/client/GatewayApi.scala | 27 +- .../elastic/client/MappingApi.scala | 17 + .../elastic/client/NopeClientApi.scala | 47 ++- .../elastic/client/TransformApi.scala | 178 ++++++++ .../client/metrics/MetricsElasticClient.scala | 63 ++- .../elastic/client/result/package.scala | 4 +- .../elastic/client/jest/JestClientApi.scala | 2 + .../client/jest/JestEnrichPolicyApi.scala | 58 +++ .../elastic/client/jest/JestMappingApi.scala | 25 ++ .../client/jest/JestTransformApi.scala | 83 ++++ .../client/rest/RestHighLevelClientApi.scala | 138 ++++++- .../client/rest/RestHighLevelClientApi.scala | 388 +++++++++++++++++- .../elastic/client/java/JavaClientApi.scala | 228 +++++++++- .../elastic/client/java/JavaClientApi.scala | 225 +++++++++- .../elastic/licensing/package.scala | 11 +- .../softnetwork/elastic/schema/package.scala | 5 + .../elastic/sql/parser/Parser.scala | 42 +- .../softnetwork/elastic/sql/query/From.scala | 2 +- .../elastic/sql/query/package.scala | 42 +- .../elastic/sql/schema/package.scala | 284 +++++++++---- .../elastic/sql/serialization/package.scala | 6 + .../client/GatewayApiIntegrationSpec.scala | 30 +- 28 files changed, 2002 insertions(+), 112 deletions(-) create mode 100644 core/src/main/scala/app/softnetwork/elastic/client/EnrichPolicyApi.scala create mode 100644 core/src/main/scala/app/softnetwork/elastic/client/TransformApi.scala create mode 100644 es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestEnrichPolicyApi.scala create mode 100644 es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTransformApi.scala diff --git a/build.sbt b/build.sbt index 54b5effc..c6ef7f6e 100644 --- a/build.sbt +++ b/build.sbt @@ -1,5 +1,6 @@ +import app.softnetwork.Publish +import scala.collection.Seq import SoftClient4es.* -import app.softnetwork.* import sbt.Def import sbtbuildinfo.BuildInfoKeys.buildInfoObject @@ -19,7 +20,7 @@ ThisBuild / organization := "app.softnetwork" name := "softclient4es" -ThisBuild / version := "0.16.0" +ThisBuild / version := "0.16-SNAPSHOT" ThisBuild / scalaVersion := scala213 diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala index 9e2f8723..c4542683 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala @@ -48,6 +48,8 @@ trait ElasticClientApi with SerializationApi with PipelineApi with TemplateApi + with EnrichPolicyApi + with TransformApi with ExtensionApi with GatewayApi with ClientCompanion { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index c5888dfb..2327ccd4 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -22,7 +22,7 @@ import akka.stream.scaladsl.{Flow, Source} import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.schema.{Index, IndexMappings} import app.softnetwork.elastic.sql.{query, schema, PainlessContextType} import app.softnetwork.elastic.sql.query.{ DqlStatement, @@ -30,7 +30,7 @@ import app.softnetwork.elastic.sql.query.{ SelectStatement, SingleSearch } -import app.softnetwork.elastic.sql.schema.{Schema, TableAlias} +import app.softnetwork.elastic.sql.schema.{Schema, TableAlias, TransformCreationStatus} import com.typesafe.config.Config import org.json4s.Formats import org.slf4j.{Logger, LoggerFactory} @@ -610,6 +610,9 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { ): ElasticResult[Boolean] = delegate.updateMapping(index, mapping, settings) + override def allMappings: ElasticResult[Map[String, IndexMappings]] = + delegate.allMappings + /** Migrate an existing index to a new mapping. * * Process: @@ -645,6 +648,10 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { delegate.executeGetMapping(index) } + override private[client] def executeGetAllMappings(): ElasticResult[Map[String, String]] = { + delegate.executeGetAllMappings() + } + // ==================== RefreshApi ==================== /** Refresh the index to make sure all documents are indexed and searchable. @@ -1798,4 +1805,82 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { system: ActorSystem ): Future[ElasticResult[QueryResult]] = delegate.run(statement) + + // ==================== Transform (delegate) ==================== + + override def createTransform( + config: schema.TransformConfig, + start: Boolean + ): ElasticResult[TransformCreationStatus] = + delegate.createTransform(config, start) + + override def deleteTransform(transformId: String, force: Boolean): ElasticResult[Boolean] = + delegate.deleteTransform(transformId, force) + + override def startTransform(transformId: String): ElasticResult[Boolean] = + delegate.startTransform(transformId) + + override def stopTransform( + transformId: String, + force: Boolean, + waitForCompletion: Boolean + ): ElasticResult[Boolean] = + delegate.stopTransform(transformId, force, waitForCompletion) + + override def getTransformStats( + transformId: String + ): ElasticResult[Option[schema.TransformStats]] = + delegate.getTransformStats(transformId) + + override private[client] def executeCreateTransform( + config: schema.TransformConfig, + start: Boolean + ): ElasticResult[Boolean] = + delegate.executeCreateTransform(config, start) + + override private[client] def executeDeleteTransform( + transformId: String, + force: Boolean + ): ElasticResult[Boolean] = + delegate.executeDeleteTransform(transformId, force) + + override private[client] def executeStartTransform(transformId: String): ElasticResult[Boolean] = + delegate.executeStartTransform(transformId) + + override private[client] def executeStopTransform( + transformId: String, + force: Boolean, + waitForCompletion: Boolean + ): ElasticResult[Boolean] = + delegate.executeStopTransform(transformId, force, waitForCompletion) + + override private[client] def executeGetTransformStats( + transformId: String + ): ElasticResult[Option[schema.TransformStats]] = + delegate.executeGetTransformStats(transformId) + // ==================== Enrich policy (delegate) ==================== + + override def createEnrichPolicy(policy: schema.EnrichPolicy): ElasticResult[Boolean] = + delegate.createEnrichPolicy(policy) + + override def deleteEnrichPolicy(policyName: String): ElasticResult[Boolean] = + delegate.deleteEnrichPolicy(policyName) + + override def executeEnrichPolicy(policyName: String): ElasticResult[String] = + delegate.executeEnrichPolicy(policyName) + + override private[client] def executeCreateEnrichPolicy( + policy: schema.EnrichPolicy + ): ElasticResult[Boolean] = + delegate.executeCreateEnrichPolicy(policy) + + override private[client] def executeDeleteEnrichPolicy( + policyName: String + ): ElasticResult[Boolean] = + delegate.executeDeleteEnrichPolicy(policyName) + + override private[client] def executeExecuteEnrichPolicy( + policyName: String + ): ElasticResult[String] = + delegate.executeExecuteEnrichPolicy(policyName) } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala index 24e9ad72..7d239c29 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala @@ -149,4 +149,22 @@ object ElasticsearchVersion { def supportsDeletionByQueryOnClosedIndices(version: String): Boolean = { isAtLeast(version, 7, 5) } + + /** Check if enrich processor is supported (ES >= 7.5) + */ + def supportsEnrich(version: String): Boolean = { + isAtLeast(version, 7, 5) + } + + /** Check if latest transform features are supported (ES >= 7.8) + */ + def supportsLatestTransform(version: String): Boolean = { + isAtLeast(version, 7, 8) + } + + /** Check if materialized views are supported (enrich + latest transform) + */ + def supportsMaterializedView(version: String): Boolean = { + supportsEnrich(version) && supportsLatestTransform(version) + } } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/EnrichPolicyApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/EnrichPolicyApi.scala new file mode 100644 index 00000000..fedda016 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/EnrichPolicyApi.scala @@ -0,0 +1,90 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.result.{ + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess +} +import app.softnetwork.elastic.sql.schema.EnrichPolicy + +trait EnrichPolicyApi extends ElasticClientHelpers { _: VersionApi => + + private def checkVersionForEnrichPolicy(): ElasticResult[Boolean] = { + val elasticVersion = version match { + case ElasticSuccess(value) => value + case ElasticFailure(error) => + val failure = + s"Cannot retrieve elastic version to check enrich policy support: ${error.getMessage}" + logger.error(s"❌ $failure") + return ElasticFailure( + ElasticError( + message = failure, + operation = Some("CheckVersionForEnrichPolicy"), + statusCode = Some(400) + ) + ) + } + if (!ElasticsearchVersion.supportsEnrich(elasticVersion)) { + val failure = s"Enrich policies are not supported in elastic version $elasticVersion" + logger.error(s"❌ $failure") + return ElasticFailure( + ElasticError( + message = failure, + operation = Some("CheckVersionForEnrichPolicy"), + statusCode = Some(400) + ) + ) + } + ElasticSuccess(true) + } + + def createEnrichPolicy(policy: EnrichPolicy): ElasticResult[Boolean] = { + checkVersionForEnrichPolicy() match { + case ElasticSuccess(_) => // continue + case failure @ ElasticFailure(_) => return failure + } + logger.info(s"🔧 Creating enrich policy: ${policy.name}") + executeCreateEnrichPolicy(policy) + } + + def deleteEnrichPolicy(policyName: String): ElasticResult[Boolean] = { + checkVersionForEnrichPolicy() match { + case ElasticSuccess(_) => // continue + case failure @ ElasticFailure(_) => return failure + } + logger.info(s"🔧 Deleting enrich policy: $policyName") + executeDeleteEnrichPolicy(policyName) + } + + def executeEnrichPolicy(policyName: String): ElasticResult[String] = { + checkVersionForEnrichPolicy() match { + case ElasticSuccess(_) => // continue + case failure @ ElasticFailure(_) => return failure + } + logger.info(s"🔧 Executing enrich policy: $policyName") + executeExecuteEnrichPolicy(policyName) + } + + private[client] def executeCreateEnrichPolicy(policy: EnrichPolicy): ElasticResult[Boolean] + + private[client] def executeDeleteEnrichPolicy(policyName: String): ElasticResult[Boolean] + + private[client] def executeExecuteEnrichPolicy(policyName: String): ElasticResult[String] +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ExtensionApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ExtensionApi.scala index 80d9eda8..56b857e9 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ExtensionApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ExtensionApi.scala @@ -18,12 +18,11 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.licensing.{DefaultLicenseManager, LicenseManager} -trait ExtensionApi { _: ElasticClientApi => +trait ExtensionApi { self: ElasticClientApi => // ✅ Inject license manager (overridable) def licenseManager: LicenseManager = new DefaultLicenseManager() /** Extension registry (lazy loaded) */ - lazy val extensionRegistry: ExtensionRegistry = - new ExtensionRegistry(config, licenseManager) + lazy val extensionRegistry: ExtensionRegistry = new ExtensionRegistry(config, licenseManager) } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala index 48e7a2d9..c5184070 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala @@ -53,6 +53,7 @@ import app.softnetwork.elastic.sql.query.{ ShowCreateTable, ShowPipeline, ShowTable, + ShowTables, SingleSearch, Statement, TableStatement, @@ -277,7 +278,7 @@ class PipelineExecutor(api: PipelineApi, logger: Logger) extends DdlExecutor[Pip case ElasticSuccess(pipeline) => logger.info(s"✅ Retrieved pipeline ${describe.name}.") Future.successful( - ElasticResult.success(QueryRows(pipeline.processors.map(_.properties))) + ElasticResult.success(QueryRows(pipeline.describe)) ) case ElasticFailure(elasticError) => Future.successful( @@ -317,6 +318,26 @@ class TableExecutor( // handle TABLE statement statement match { // handle SHOW TABLE statement + case ShowTables => + api.allMappings match { + case ElasticSuccess(mappings) => + logger.info("✅ Retrieved all tables.") + Future.successful( + ElasticResult.success( + QueryRows( + mappings.map { case (index, mappings) => + Map("name" -> index, "type" -> mappings.tableType.name.toUpperCase) + }.toSeq + ) + ) + ) + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("tables")) + ) + ) + } case show: ShowTable => api.loadSchema(show.table) match { case ElasticSuccess(schema) => @@ -348,7 +369,7 @@ class TableExecutor( api.loadSchema(describe.table) match { case ElasticSuccess(schema) => logger.info(s"✅ Retrieved schema for index ${describe.table}.") - Future.successful(ElasticResult.success(QueryRows(schema.columns.flatMap(_.asMap)))) + Future.successful(ElasticResult.success(QueryRows(schema.describe))) case ElasticFailure(elasticError) => Future.successful( ElasticFailure( @@ -408,7 +429,6 @@ class TableExecutor( // 4) Index not exists → creation case ElasticSuccess(false) => // proceed with creation - logger.info(s"✅ Creating index $indexName.") createNonExistentIndex(indexName, create, partitioned, single) // 5) Error on indexExists @@ -915,7 +935,6 @@ class TableExecutor( ) match { case success @ ElasticSuccess(true) => // index created successfully - logger.info(s"✅ Index $indexName created successfully.") success case ElasticSuccess(_) => // index creation failed diff --git a/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala index 088d0bb0..5c878125 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala @@ -22,6 +22,7 @@ import app.softnetwork.elastic.client.result.{ ElasticResult, ElasticSuccess } +import app.softnetwork.elastic.schema.IndexMappings import app.softnetwork.elastic.sql.schema.TableAlias import com.fasterxml.jackson.databind.node.ObjectNode import com.google.gson.JsonParser @@ -154,6 +155,20 @@ trait MappingApi extends ElasticClientHelpers { executeGetMapping(index) } + /** Get the mappings of all indices. + * @return + * a map of index names to their mappings + */ + def allMappings: ElasticResult[Map[String, IndexMappings]] = { + // Get mappings for all indices + executeGetAllMappings().map { mappingsMap => + mappingsMap.map { case (index, mappingJson) => + val indexMappings = IndexMappings(mappingJson) + (index, indexMappings) + } + } + } + /** Get the mapping properties of an index. * @param index * - the name of the index to get the mapping properties for @@ -392,4 +407,6 @@ trait MappingApi extends ElasticClientHelpers { private[client] def executeSetMapping(index: String, mapping: String): ElasticResult[Boolean] private[client] def executeGetMapping(index: String): ElasticResult[String] + + private[client] def executeGetAllMappings(): ElasticResult[Map[String, String]] } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala index 91151291..f1f0afa0 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala @@ -26,7 +26,7 @@ import app.softnetwork.elastic.client.bulk.{ FailedDocument, SuccessfulDocument } -import app.softnetwork.elastic.sql.{query, PainlessContextType} +import app.softnetwork.elastic.sql.{query, schema, PainlessContextType} import app.softnetwork.elastic.sql.query.SQLAggregation import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ @@ -379,4 +379,49 @@ trait NopeClientApi extends ElasticClientApi { /** Conversion BulkActionType -> BulkItem */ override private[client] def actionToBulkItem(action: BulkActionType): BulkItem = throw new UnsupportedOperationException + + override private[client] def executeGetAllMappings(): ElasticResult[Map[String, String]] = + ElasticResult.success(Map.empty) + + override private[client] def executeCreateEnrichPolicy( + policy: schema.EnrichPolicy + ): ElasticResult[Boolean] = + ElasticSuccess(false) + + override private[client] def executeDeleteEnrichPolicy( + policyName: String + ): ElasticResult[Boolean] = + ElasticSuccess(false) + + override private[client] def executeExecuteEnrichPolicy( + policyName: String + ): ElasticResult[String] = + ElasticSuccess("") + + override private[client] def executeCreateTransform( + config: schema.TransformConfig, + start: Boolean + ): ElasticResult[Boolean] = + ElasticSuccess(false) + + override private[client] def executeDeleteTransform( + transformId: String, + force: Boolean + ): ElasticResult[Boolean] = + ElasticSuccess(false) + + override private[client] def executeStartTransform(transformId: String): ElasticResult[Boolean] = + ElasticSuccess(false) + + override private[client] def executeStopTransform( + transformId: String, + force: Boolean, + waitForCompletion: Boolean + ): ElasticResult[Boolean] = + ElasticSuccess(false) + + override private[client] def executeGetTransformStats( + transformId: String + ): ElasticResult[Option[schema.TransformStats]] = + ElasticSuccess(None) } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/TransformApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/TransformApi.scala new file mode 100644 index 00000000..e15713d2 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/TransformApi.scala @@ -0,0 +1,178 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.result.{ + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess +} +import app.softnetwork.elastic.sql.schema.{TransformConfig, TransformCreationStatus, TransformStats} + +trait TransformApi extends ElasticClientHelpers { + + /** Create a transform with the given configuration. If start is true, the transform will be + * started immediately after creation. + * @param config + * The configuration of the transform to create. + * @param start + * Whether to start the transform immediately after creation. + * @return + * ElasticResult[TransformCreationStatus] indicating whether the create operation and its + * optional start were successful. + */ + def createTransform( + config: TransformConfig, + start: Boolean = false + ): ElasticResult[TransformCreationStatus] = { + logger.info(s"Creating transform [${config.id}]...") + executeCreateTransform(config, start) match { + case ElasticSuccess(true) if start => + logger.info(s"✅ Transform [${config.id}] created successfully. Starting transform...") + startTransform(config.id) match { + case ElasticSuccess(status) => + ElasticSuccess(TransformCreationStatus(created = true, started = Some(status))) + case failure @ ElasticFailure(_) => + failure + } + case ElasticSuccess(status) => + ElasticSuccess(TransformCreationStatus(created = status)) + case failure @ ElasticFailure(_) => failure + } + } + + /** Delete the transform with the given ID. If force is true, the transform will be deleted even + * if it is currently running. + * @param transformId + * The ID of the transform to delete. + * @param force + * Whether to force delete the transform. + * @return + * ElasticResult[Boolean] indicating whether the delete operation was successful. + */ + def deleteTransform(transformId: String, force: Boolean = false): ElasticResult[Boolean] = { + logger.info(s"Deleting transform [$transformId]...") + executeDeleteTransform(transformId, force) match { + case success @ ElasticSuccess(true) => + logger.info(s"✅ Transform [$transformId] deleted successfully.") + success + case success @ ElasticSuccess(false) => + logger.warn(s"⚠️ Transform [$transformId] could not be deleted.") + success + case failure @ ElasticFailure(error) => + logger.error(s"❌ Failed to delete transform [$transformId]: ${error.message}") + failure + } + } + + /** Start the transform with the given ID. + * @param transformId + * The ID of the transform to start. + * @return + * ElasticResult[Boolean] indicating whether the start operation was successful. + */ + def startTransform(transformId: String): ElasticResult[Boolean] = { + logger.info(s"Starting transform [$transformId]...") + executeStartTransform(transformId) match { + case success @ ElasticSuccess(true) => + logger.info(s"✅ Transform [$transformId] started successfully.") + success + case success @ ElasticSuccess(false) => + logger.warn(s"⚠️ Transform [$transformId] could not be started.") + success + case failure @ ElasticFailure(error) => + logger.error(s"❌ Failed to start transform [$transformId]: ${error.message}") + failure + } + } + + /** Stop the transform with the given ID. If force is true, the transform will be stopped + * immediately. If waitForCompletion is true, the method will wait until the transform is fully + * stopped before returning. + * @param transformId + * The ID of the transform to stop. + * @param force + * Whether to force stop the transform. + * @param waitForCompletion + * Whether to wait for the transform to be fully stopped. + * @return + * ElasticResult[Boolean] indicating whether the stop operation was successful. + */ + def stopTransform( + transformId: String, + force: Boolean = false, + waitForCompletion: Boolean = true + ): ElasticResult[Boolean] = { + logger.info(s"Stopping transform [$transformId]...") + executeStopTransform(transformId, force, waitForCompletion) match { + case success @ ElasticSuccess(true) => + logger.info(s"✅ Transform [$transformId] stopped successfully.") + success + case success @ ElasticSuccess(false) => + logger.warn(s"⚠️ Transform [$transformId] could not be stopped.") + success + case failure @ ElasticFailure(error) => + logger.error(s"❌ Failed to stop transform [$transformId]: ${error.message}") + failure + } + } + + /** Get the statistics of the transform with the given ID. + * @param transformId + * The ID of the transform to get statistics for. + * @return + * ElasticResult[Option[TransformStats]\] containing the transform statistics if available. + */ + def getTransformStats(transformId: String): ElasticResult[Option[TransformStats]] = { + logger.info(s"Getting stats for transform [$transformId]...") + executeGetTransformStats(transformId) match { + case success @ ElasticSuccess(Some(_)) => + logger.info(s"✅ Retrieved stats for transform [$transformId] successfully.") + success + case success @ ElasticSuccess(None) => + logger.warn(s"⚠️ No stats found for transform [$transformId].") + success + case failure @ ElasticFailure(error) => + logger.error(s"❌ Failed to get stats for transform [$transformId]: ${error.message}") + failure + } + } + + private[client] def executeCreateTransform( + config: TransformConfig, + start: Boolean + ): ElasticResult[Boolean] + + private[client] def executeDeleteTransform( + transformId: String, + force: Boolean + ): ElasticResult[Boolean] + + private[client] def executeStartTransform(transformId: String): ElasticResult[Boolean] + + private[client] def executeStopTransform( + transformId: String, + force: Boolean, + waitForCompletion: Boolean + ): ElasticResult[Boolean] + + private[client] def executeGetTransformStats( + transformId: String + ): ElasticResult[Option[TransformStats]] + +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala index 7b78425f..edf0af60 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala @@ -30,10 +30,10 @@ import app.softnetwork.elastic.client.{ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.schema.{Index, IndexMappings} import app.softnetwork.elastic.sql.{query, schema} import app.softnetwork.elastic.sql.query.{DqlStatement, SQLAggregation, SelectStatement} -import app.softnetwork.elastic.sql.schema.{Schema, TableAlias} +import app.softnetwork.elastic.sql.schema.{Schema, TableAlias, TransformCreationStatus} import org.json4s.Formats import scala.concurrent.{ExecutionContext, Future} @@ -454,6 +454,11 @@ class MetricsElasticClient( delegate.updateMapping(index, mapping, settings) } + override def allMappings: ElasticResult[Map[String, IndexMappings]] = + measureResult("allMappings") { + delegate.allMappings + } + // ==================== RefreshApi ==================== override def refresh(index: String): ElasticResult[Boolean] = { @@ -1365,4 +1370,58 @@ class MetricsElasticClient( } } + // ==================== Transform (delegate) ==================== + + override def createTransform( + config: schema.TransformConfig, + start: Boolean + ): ElasticResult[TransformCreationStatus] = + measureResult("createTransform") { + delegate.createTransform(config, start) + } + + override def deleteTransform(transformId: String, force: Boolean): ElasticResult[Boolean] = + measureResult("deleteTransform") { + delegate.deleteTransform(transformId, force) + } + + override def startTransform(transformId: String): ElasticResult[Boolean] = { + measureResult("startTransform") { + delegate.startTransform(transformId) + } + } + + override def stopTransform( + transformId: String, + force: Boolean, + waitForCompletion: Boolean + ): ElasticResult[Boolean] = + measureResult("stopTransform") { + delegate.stopTransform(transformId, force, waitForCompletion) + } + + override def getTransformStats( + transformId: String + ): ElasticResult[Option[schema.TransformStats]] = + measureResult("getTransformStats") { + delegate.getTransformStats(transformId) + } + + // ==================== Enrich policy (delegate) ==================== + + override def createEnrichPolicy(policy: schema.EnrichPolicy): ElasticResult[Boolean] = + measureResult("createEnrichPolicy") { + delegate.createEnrichPolicy(policy) + } + + override def deleteEnrichPolicy(policyName: String): ElasticResult[Boolean] = + measureResult("deleteEnrichPolicy") { + delegate.deleteEnrichPolicy(policyName) + } + + override def executeEnrichPolicy(policyName: String): ElasticResult[String] = + measureResult("executeEnrichPolicy") { + delegate.executeEnrichPolicy(policyName) + } + } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala index 3582acfe..91362e78 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala @@ -401,7 +401,9 @@ package object result { stream: Source[(Map[String, Any], ScrollMetrics), NotUsed] ) extends QueryResult - case class QueryStructured(response: ElasticResponse) extends QueryResult + case class QueryStructured(response: ElasticResponse) extends QueryResult { + def asQueryRows: QueryRows = QueryRows(response.results) + } // -------------------- // DML (INSERT / UPDATE / DELETE) diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala index 9a31eaa1..bf607664 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala @@ -46,6 +46,8 @@ trait JestClientApi with JestVersionApi with JestPipelineApi with JestTemplateApi + with JestEnrichPolicyApi + with JestTransformApi with JestClientCompanion object JestClientApi extends SerializationApi { diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestEnrichPolicyApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestEnrichPolicyApi.scala new file mode 100644 index 00000000..e1364edf --- /dev/null +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestEnrichPolicyApi.scala @@ -0,0 +1,58 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.jest + +import app.softnetwork.elastic.client.result.ElasticFailure +import app.softnetwork.elastic.client.{result, EnrichPolicyApi} +import app.softnetwork.elastic.sql.schema + +trait JestEnrichPolicyApi extends EnrichPolicyApi with JestClientHelpers { + _: JestVersionApi with JestClientCompanion => + + override private[client] def executeCreateEnrichPolicy( + policy: schema.EnrichPolicy + ): result.ElasticResult[Boolean] = + ElasticFailure( + result.ElasticError( + message = "Enrich policy creation not implemented for Jest client", + operation = Some("CreateEnrichPolicy"), + statusCode = Some(501) + ) + ) + + override private[client] def executeDeleteEnrichPolicy( + policyName: String + ): result.ElasticResult[Boolean] = + ElasticFailure( + result.ElasticError( + message = "Enrich policy deletion not implemented for Jest client", + operation = Some("CreateEnrichPolicy"), + statusCode = Some(501) + ) + ) + + override private[client] def executeExecuteEnrichPolicy( + policyName: String + ): result.ElasticResult[String] = + ElasticFailure( + result.ElasticError( + message = "Enrich policy execution not implemented for Jest client", + operation = Some("ExecuteEnrichPolicy"), + statusCode = Some(501) + ) + ) +} diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala index fe53c9c3..a55e9580 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala @@ -102,4 +102,29 @@ trait JestMappingApi extends MappingApi with JestClientHelpers { } } } + + override private[client] def executeGetAllMappings(): ElasticResult[Map[String, String]] = + executeJestAction( + operation = "getAllMappings", + index = None, + retryable = true + )( + new GetMapping.Builder().build() + ) { result => + val jsonString = result.getJsonString + val jsonObject = JsonParser.parseString(jsonString).getAsJsonObject + val mappings = jsonObject + .entrySet() + .toArray + .map { entry => + val e = entry.asInstanceOf[java.util.Map.Entry[String, com.google.gson.JsonElement]] + val indexName = e.getKey + val mappingObj = e.getValue.getAsJsonObject + .getAsJsonObject("mappings") + .getAsJsonObject("_doc") + indexName -> mappingObj.toString + } + .toMap + mappings + } } diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTransformApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTransformApi.scala new file mode 100644 index 00000000..c76855df --- /dev/null +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTransformApi.scala @@ -0,0 +1,83 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.jest + +import app.softnetwork.elastic.client.{result, TransformApi} +import app.softnetwork.elastic.sql.schema + +trait JestTransformApi extends TransformApi with JestClientHelpers { + _: JestClientCompanion => + + override private[client] def executeCreateTransform( + config: schema.TransformConfig, + start: Boolean + ): result.ElasticResult[Boolean] = + result.ElasticFailure( + result.ElasticError( + message = "Transform creation not implemented for Jest client", + operation = Some("CreateTransform"), + statusCode = Some(501) + ) + ) + + override private[client] def executeDeleteTransform( + transformId: String, + force: Boolean + ): result.ElasticResult[Boolean] = + result.ElasticFailure( + result.ElasticError( + message = "Transform deletion not implemented for Jest client", + operation = Some("DeleteTransform"), + statusCode = Some(501) + ) + ) + + override private[client] def executeStartTransform( + transformId: String + ): result.ElasticResult[Boolean] = + result.ElasticFailure( + result.ElasticError( + message = "Transform start not implemented for Jest client", + operation = Some("StartTransform"), + statusCode = Some(501) + ) + ) + + override private[client] def executeStopTransform( + transformId: String, + force: Boolean, + waitForCompletion: Boolean + ): result.ElasticResult[Boolean] = + result.ElasticFailure( + result.ElasticError( + message = "Transform stop not implemented for Jest client", + operation = Some("StopTransform"), + statusCode = Some(501) + ) + ) + + override private[client] def executeGetTransformStats( + transformId: String + ): result.ElasticResult[Option[schema.TransformStats]] = + result.ElasticFailure( + result.ElasticError( + message = "Get Transform stats not implemented for Jest client", + operation = Some("GetTransformStats"), + statusCode = Some(501) + ) + ) +} diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index a3a28bef..b858aaec 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -21,12 +21,17 @@ import akka.actor.ActorSystem import akka.stream.scaladsl.{Flow, Source} import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ -import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} +import app.softnetwork.elastic.client.result.{ + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess +} import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.sql.{ObjectValue, PainlessContextType, Value} +import app.softnetwork.elastic.sql.{schema, ObjectValue, PainlessContextType, Value} import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.schema.TableAlias +import app.softnetwork.elastic.sql.schema.{EnrichPolicy, TableAlias} import com.fasterxml.jackson.databind.node.ObjectNode import com.google.gson.JsonParser import org.apache.http.util.EntityUtils @@ -114,6 +119,8 @@ trait RestHighLevelClientApi with RestHighLevelClientVersionApi with RestHighLevelClientPipelineApi with RestHighLevelClientTemplateApi + with RestHighLevelClientEnrichPolicyApi + with RestHighLevelClientTransformApi /** Version API implementation for RestHighLevelClient * @see @@ -608,6 +615,28 @@ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientH } }) + override private[client] def executeGetAllMappings(): ElasticResult[Map[String, String]] = + executeRestAction[ + GetMappingsRequest, + GetMappingsResponse, + Map[String, String] + ]( + operation = "getAllMappings", + index = None, + retryable = true + )( + request = new GetMappingsRequest().indices() + )( + executor = req => apply().indices().getMapping(req, RequestOptions.DEFAULT) + )(response => { + response + .mappings() + .asScala + .map { case (index, metadata) => + (index, metadata.source().toString) + } + .toMap + }) } /** Refresh API implementation for RestHighLevelClient @@ -1900,3 +1929,106 @@ trait RestHighLevelClientTemplateApi extends TemplateApi with RestHighLevelClien } } } + +// ==================== ENRICH POLICY API IMPLEMENTATION FOR REST HIGH LEVEL CLIENT ==================== +trait RestHighLevelClientEnrichPolicyApi extends EnrichPolicyApi with RestHighLevelClientHelpers { + _: RestHighLevelClientVersionApi with RestHighLevelClientCompanion => + + override private[client] def executeCreateEnrichPolicy( + policy: EnrichPolicy + ): result.ElasticResult[Boolean] = + ElasticFailure( + result.ElasticError( + message = "Enrich policy creation not implemented for Rest client", + operation = Some("CreateEnrichPolicy"), + statusCode = Some(501) + ) + ) + + override private[client] def executeDeleteEnrichPolicy( + policyName: String + ): result.ElasticResult[Boolean] = + ElasticFailure( + result.ElasticError( + message = "Enrich policy deletion not implemented for Rest client", + operation = Some("CreateEnrichPolicy"), + statusCode = Some(501) + ) + ) + + override private[client] def executeExecuteEnrichPolicy( + policyName: String + ): ElasticResult[String] = + ElasticFailure( + result.ElasticError( + message = "Enrich policy execution not implemented for Rest client", + operation = Some("ExecuteEnrichPolicy"), + statusCode = Some(501) + ) + ) +} + +// ==================== TRANSFORM API IMPLEMENTATION FOR REST HIGH LEVEL CLIENT ==================== + +trait RestHighLevelClientTransformApi extends TransformApi with RestHighLevelClientHelpers { + _: RestHighLevelClientVersionApi with RestHighLevelClientCompanion with SerializationApi => + + override private[client] def executeCreateTransform( + config: schema.TransformConfig, + start: Boolean + ): result.ElasticResult[Boolean] = + result.ElasticFailure( + result.ElasticError( + message = "Transform creation not implemented for Rest client", + operation = Some("CreateTransform"), + statusCode = Some(501) + ) + ) + + override private[client] def executeDeleteTransform( + transformId: String, + force: Boolean + ): result.ElasticResult[Boolean] = + result.ElasticFailure( + result.ElasticError( + message = "Transform deletion not implemented for Rest client", + operation = Some("DeleteTransform"), + statusCode = Some(501) + ) + ) + + override private[client] def executeStartTransform( + transformId: String + ): result.ElasticResult[Boolean] = + result.ElasticFailure( + result.ElasticError( + message = "Transform start not implemented for Rest client", + operation = Some("StartTransform"), + statusCode = Some(501) + ) + ) + + override private[client] def executeStopTransform( + transformId: String, + force: Boolean, + waitForCompletion: Boolean + ): result.ElasticResult[Boolean] = + result.ElasticFailure( + result.ElasticError( + message = "Transform stop not implemented for Rest client", + operation = Some("StopTransform"), + statusCode = Some(501) + ) + ) + + override private[client] def executeGetTransformStats( + transformId: String + ): result.ElasticResult[Option[schema.TransformStats]] = + result.ElasticFailure( + result.ElasticError( + message = "Get Transform stats not implemented for Rest client", + operation = Some("GetTransformStats"), + statusCode = Some(501) + ) + ) +} diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 52c2db53..78aaba78 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -23,11 +23,24 @@ import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.sql.{ObjectValue, PainlessContextType, Value} +import app.softnetwork.elastic.sql.{schema, ObjectValue, PainlessContextType, Value} import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} -import app.softnetwork.elastic.sql.schema.TableAlias +import app.softnetwork.elastic.sql.query.{Asc, Criteria, Desc, SQLAggregation, SingleSearch} +import app.softnetwork.elastic.sql.schema.{ + AvgTransformAggregation, + CardinalityTransformAggregation, + CountTransformAggregation, + EnrichPolicy, + MaxTransformAggregation, + MinTransformAggregation, + SumTransformAggregation, + TableAlias, + TermsGroupBy, + TopHitsTransformAggregation, + TransformState +} import app.softnetwork.elastic.sql.serialization.JacksonConfig +import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.ObjectNode import com.google.gson.JsonParser import org.apache.http.util.EntityUtils @@ -68,6 +81,7 @@ import org.elasticsearch.action.update.{UpdateRequest, UpdateResponse} import org.elasticsearch.action.{ActionListener, DocWriteRequest, DocWriteResponse} import org.elasticsearch.client.{GetAliasesResponse, Request, RequestOptions, Response} import org.elasticsearch.client.core.{CountRequest, CountResponse} +import org.elasticsearch.client.enrich.{DeletePolicyRequest, ExecutePolicyRequest, PutPolicyRequest} import org.elasticsearch.client.indices.{ CloseIndexRequest, ComposableIndexTemplateExistRequest, @@ -84,12 +98,45 @@ import org.elasticsearch.client.indices.{ PutIndexTemplateRequest, PutMappingRequest } +import org.elasticsearch.client.transform.transforms.latest.LatestConfig +import org.elasticsearch.client.transform.{ + DeleteTransformRequest, + GetTransformStatsRequest, + PutTransformRequest, + StartTransformRequest, + StopTransformRequest +} +import org.elasticsearch.client.transform.transforms.pivot.{ + AggregationConfig, + GroupConfig, + PivotConfig, + TermsGroupSource +} +import org.elasticsearch.client.transform.transforms.{ + DestConfig, + QueryConfig, + SourceConfig, + TimeSyncConfig, + TransformConfig +} import org.elasticsearch.cluster.metadata.{AliasMetadata, ComposableIndexTemplate} import org.elasticsearch.common.Strings import org.elasticsearch.common.bytes.BytesArray import org.elasticsearch.core.TimeValue +import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.xcontent.{DeprecationHandler, ToXContent, XContentFactory, XContentType} import org.elasticsearch.rest.RestStatus +import org.elasticsearch.search.aggregations.AggregatorFactories +import org.elasticsearch.search.aggregations.metrics.{ + AvgAggregationBuilder, + CardinalityAggregationBuilder, + MaxAggregationBuilder, + MinAggregationBuilder, + SumAggregationBuilder, + TopHitsAggregationBuilder, + ValueCountAggregationBuilder +} +import org.elasticsearch.search.aggregations.pipeline.BucketSelectorPipelineAggregationBuilder import org.elasticsearch.search.builder.{PointInTimeBuilder, SearchSourceBuilder} import org.elasticsearch.search.sort.{FieldSortBuilder, SortOrder} import org.json4s.jackson.JsonMethods @@ -121,6 +168,8 @@ trait RestHighLevelClientApi with RestHighLevelClientVersionApi with RestHighLevelClientPipelineApi with RestHighLevelClientTemplateApi + with RestHighLevelClientEnrichPolicyApi + with RestHighLevelClientTransformApi /** Version API implementation for RestHighLevelClient * @see @@ -597,6 +646,28 @@ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientH } }) + override private[client] def executeGetAllMappings(): ElasticResult[Map[String, String]] = + executeRestAction[ + GetMappingsRequest, + org.elasticsearch.client.indices.GetMappingsResponse, + Map[String, String] + ]( + operation = "getAllMappings", + index = None, + retryable = true + )( + request = new GetMappingsRequest().indices() + )( + executor = req => apply().indices().getMapping(req, RequestOptions.DEFAULT) + )(response => { + response + .mappings() + .asScala + .map { case (index, mappings) => + (index, mappings.source().toString) + } + .toMap + }) } /** Refresh API implementation for RestHighLevelClient @@ -2167,3 +2238,314 @@ trait RestHighLevelClientTemplateApi extends TemplateApi with RestHighLevelClien } } } + +// ==================== ENRICH POLICY API IMPLEMENTATION FOR REST HIGH LEVEL CLIENT ==================== + +trait RestHighLevelClientEnrichPolicyApi extends EnrichPolicyApi with RestHighLevelClientHelpers { + _: RestHighLevelClientVersionApi with RestHighLevelClientCompanion => + + override private[client] def executeCreateEnrichPolicy( + policy: EnrichPolicy + ): result.ElasticResult[Boolean] = + executeRestAction( + operation = "CreateEnrichPolicy", + retryable = false + )( + request = { + new PutPolicyRequest( + policy.name, + policy.policyType.name.toLowerCase, + policy.indices.asJava, + policy.matchField, + policy.enrichFields.asJava + ) + } + )( + executor = req => apply().enrich().putPolicy(req, RequestOptions.DEFAULT) + )( + transformer = resp => resp.isAcknowledged + ) + + override private[client] def executeDeleteEnrichPolicy( + policyName: String + ): result.ElasticResult[Boolean] = + executeRestAction( + operation = "DeleteEnrichPolicy", + retryable = false + )( + request = new DeletePolicyRequest(policyName) + )( + executor = req => apply().enrich().deletePolicy(req, RequestOptions.DEFAULT) + )( + transformer = resp => resp.isAcknowledged + ) + + override private[client] def executeExecuteEnrichPolicy( + policyName: String + ): ElasticResult[String] = + executeRestAction( + operation = "ExecuteEnrichPolicy", + retryable = false + )( + request = { + val req = new ExecutePolicyRequest(policyName) + req.setWaitForCompletion(true) + req + } + )( + executor = req => apply().enrich().executePolicy(req, RequestOptions.DEFAULT) + )( + transformer = resp => resp.getTaskId + ) +} + +// ==================== TRANSFORM API IMPLEMENTATION FOR REST HIGH LEVEL CLIENT ==================== + +trait RestHighLevelClientTransformApi extends TransformApi with RestHighLevelClientHelpers { + _: RestHighLevelClientVersionApi with RestHighLevelClientCompanion with SerializationApi => + + override private[client] def executeCreateTransform( + config: schema.TransformConfig, + start: Boolean + ): result.ElasticResult[Boolean] = { + executeRestAction( + operation = "createTransform", + retryable = false + )( + request = { + implicit val timestamp: Long = System.currentTimeMillis() + implicit val context: PainlessContextType = PainlessContextType.Transform + val conf = convertToElasticTransformConfig(config) + val req = new PutTransformRequest(conf) + req.setDeferValidation(false) + req + } + )( + executor = req => apply().transform().putTransform(req, RequestOptions.DEFAULT) + )( + transformer = resp => resp.isAcknowledged + ) + } + + private def convertToElasticTransformConfig( + config: schema.TransformConfig + )(implicit criteriaToNode: Criteria => JsonNode): TransformConfig = { + val builder = TransformConfig.builder() + + // Set ID + builder.setId(config.id) + + // Set description + builder.setDescription(config.description) + + // Set source + val sourceConfig = SourceConfig + .builder() + .setIndex(config.source.index: _*) + + config.source.query.foreach { criteria => + // Convert Criteria to QueryBuilder + val node: JsonNode = criteria + val queryJson = mapper.writeValueAsString(node) + sourceConfig.setQueryConfig( + new QueryConfig(QueryBuilders.wrapperQuery(queryJson)) + ) + } + builder.setSource(sourceConfig.build()) + + // Set destination + val destConfig = DestConfig + .builder() + .setIndex(config.dest.index) + config.dest.pipeline.foreach(destConfig.setPipeline) + builder.setDest(destConfig.build()) + + // Set frequency + builder.setFrequency( + org.elasticsearch.core.TimeValue.parseTimeValue( + config.frequency.toTransformFormat, + "frequency" + ) + ) + + // Set sync (if present) + config.sync.foreach { sync => + val syncConfig = TimeSyncConfig + .builder() + .setField(sync.time.field) + .setDelay( + org.elasticsearch.core.TimeValue.parseTimeValue( + sync.time.delay.toTransformFormat, + "delay" + ) + ) + .build() + builder.setSyncConfig(syncConfig) + } + + // Set pivot (if present) + config.pivot.foreach { pivot => + val pivotConfig = PivotConfig.builder() + + // Group by + val groupConfig = GroupConfig.builder() + pivot.groupBy.foreach { case (name, gb) => + gb match { + case TermsGroupBy(field) => + groupConfig + .groupBy(name, TermsGroupSource.builder().setField(field).build()) + } + } + pivotConfig.setGroups(groupConfig.build()) + + // Aggregations + val aggBuilder = AggregatorFactories.builder() + pivot.aggregations.foreach { case (name, agg) => + agg match { + case MaxTransformAggregation(field) => + aggBuilder.addAggregator(new MaxAggregationBuilder(name).field(field)) + case MinTransformAggregation(field) => + aggBuilder.addAggregator(new MinAggregationBuilder(name).field(field)) + case SumTransformAggregation(field) => + aggBuilder.addAggregator(new SumAggregationBuilder(name).field(field)) + case AvgTransformAggregation(field) => + aggBuilder.addAggregator(new AvgAggregationBuilder(name).field(field)) + case CountTransformAggregation(field) => + aggBuilder.addAggregator(new ValueCountAggregationBuilder(name).field(field)) + case CardinalityTransformAggregation(field) => + aggBuilder.addAggregator(new CardinalityAggregationBuilder(name).field(field)) + case TopHitsTransformAggregation(fields, size, sortFields) => + val topHitsBuilder = new TopHitsAggregationBuilder(name) + .size(size) + .fetchSource(fields.toArray, Array.empty[String]) + sortFields.foreach { sortField => + val sortOrder = sortField.order.getOrElse(Desc) match { + case Asc => SortOrder.ASC + case Desc => SortOrder.DESC + case _ => SortOrder.DESC + } + topHitsBuilder.sort(sortField.name, sortOrder) + } + aggBuilder.addAggregator(topHitsBuilder) + case _ => + throw new UnsupportedOperationException(s"Unsupported aggregation: $agg") + } + } + pivot.bucketSelector foreach { bs => + val bucketSelector = new BucketSelectorPipelineAggregationBuilder( + bs.name, + bs.bucketsPath.asJava, + new org.elasticsearch.script.Script(bs.script) + ) + aggBuilder.addPipelineAggregator(bucketSelector) + } + pivotConfig.setAggregationConfig(new AggregationConfig(aggBuilder)) + builder.setPivotConfig(pivotConfig.build()) + } + + config.latest.foreach { latest => + val latestConfig = LatestConfig + .builder() + .setUniqueKey(latest.uniqueKey.asJava) + .setSort(latest.sort) + .build() + + builder.setLatestConfig(latestConfig) + } + + builder.setMetadata(config.metadata.asJava) + + builder.build() + } + + override private[client] def executeDeleteTransform( + transformId: String, + force: Boolean + ): result.ElasticResult[Boolean] = { + executeRestAction( + operation = "deleteTransform", + retryable = false + )( + request = { + val req = new DeleteTransformRequest(transformId) + req.setForce(force) + req + } + )( + executor = req => apply().transform().deleteTransform(req, RequestOptions.DEFAULT) + )( + transformer = resp => resp.isAcknowledged + ) + } + + override private[client] def executeStartTransform( + transformId: String + ): result.ElasticResult[Boolean] = { + executeRestAction( + operation = "startTransform", + retryable = false + )( + request = new StartTransformRequest(transformId) + )( + executor = req => apply().transform().startTransform(req, RequestOptions.DEFAULT) + )( + transformer = resp => resp.isAcknowledged + ) + } + + override private[client] def executeStopTransform( + transformId: String, + force: Boolean, + waitForCompletion: Boolean + ): result.ElasticResult[Boolean] = { + executeRestAction( + operation = "stopTransform", + retryable = false + )( + request = { + val req = new StopTransformRequest(transformId) + req.setWaitForCheckpoint(!force) + req.setWaitForCompletion(waitForCompletion) + req + } + )( + executor = req => apply().transform().stopTransform(req, RequestOptions.DEFAULT) + )( + transformer = resp => resp.isAcknowledged + ) + } + + override private[client] def executeGetTransformStats( + transformId: String + ): result.ElasticResult[Option[schema.TransformStats]] = { + executeRestAction( + operation = "getTransformStats", + retryable = true + )( + request = new GetTransformStatsRequest(transformId) + )( + executor = req => apply().transform().getTransformStats(req, RequestOptions.DEFAULT) + )( + transformer = resp => { + val statsArray = resp.getTransformsStats.asScala + statsArray.headOption.map { stats => + schema.TransformStats( + id = stats.getId, + state = TransformState(stats.getState.value()), + documentsProcessed = stats.getIndexerStats.getDocumentsProcessed, + documentsIndexed = stats.getIndexerStats.getDocumentsIndexed, + indexFailures = stats.getIndexerStats.getIndexFailures, + searchFailures = stats.getIndexerStats.getSearchFailures, + lastCheckpoint = Option(stats.getCheckpointingInfo) + .flatMap(c => Option(c.getLast)) + .map(_.getCheckpoint), + operationsBehind = Option(stats.getCheckpointingInfo) + .flatMap(info => Option(info.getOperationsBehind)) + .getOrElse(0L), + processingTimeMs = stats.getIndexerStats.getProcessingTime + ) + } + } + ) + } +} diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index e8ddd8f9..b9c5862a 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -24,16 +24,16 @@ import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} +import app.softnetwork.elastic.sql.query.{Criteria, SQLAggregation, SingleSearch} +import app.softnetwork.elastic.sql.serialization._ import app.softnetwork.elastic.client.result.{ ElasticError, ElasticFailure, ElasticResult, ElasticSuccess } -import app.softnetwork.elastic.sql.PainlessContextType -import app.softnetwork.elastic.sql.schema.TableAlias -import app.softnetwork.elastic.sql.serialization._ +import app.softnetwork.elastic.sql.{schema, PainlessContextType} +import app.softnetwork.elastic.sql.schema.{EnrichPolicy, TableAlias, TransformState} import co.elastic.clients.elasticsearch._types.mapping.TypeMapping import co.elastic.clients.elasticsearch._types.{ Conflicts, @@ -61,6 +61,11 @@ import co.elastic.clients.elasticsearch.core.msearch.{ import co.elastic.clients.elasticsearch.core._ import co.elastic.clients.elasticsearch.core.reindex.{Destination, Source => ESSource} import co.elastic.clients.elasticsearch.core.search.PointInTimeReference +import co.elastic.clients.elasticsearch.enrich.{ + DeletePolicyRequest, + ExecutePolicyRequest, + PutPolicyRequest +} import co.elastic.clients.elasticsearch.indices.update_aliases.{Action, AddAction, RemoveAction} import co.elastic.clients.elasticsearch.indices.{ExistsRequest => IndexExistsRequest, _} import co.elastic.clients.elasticsearch.ingest.{ @@ -68,6 +73,13 @@ import co.elastic.clients.elasticsearch.ingest.{ GetPipelineRequest, PutPipelineRequest } +import co.elastic.clients.elasticsearch.transform.{ + DeleteTransformRequest, + GetTransformStatsRequest, + PutTransformRequest, + StartTransformRequest, + StopTransformRequest +} import com.fasterxml.jackson.databind.JsonNode import com.google.gson.JsonParser @@ -98,6 +110,8 @@ trait JavaClientApi with JavaClientVersionApi with JavaClientPipelineApi with JavaClientTemplateApi + with JavaClientEnrichPolicyApi + with JavaClientTransformApi /** Elasticsearch client implementation using the Java Client * @see @@ -559,6 +573,27 @@ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { } } } + + override private[client] def executeGetAllMappings(): ElasticResult[Map[String, String]] = + executeJavaAction( + operation = "getAllMappings", + index = None, + retryable = true + )( + apply() + .indices() + .getMapping( + new GetMappingRequest.Builder().build() + ) + ) { response => + response + .result() + .asScala + .map { case (index, mapping) => + (index, convertToJson(mapping)) + } + .toMap + } } /** Elasticsearch client implementation of Refresh API using the Java Client @@ -1952,3 +1987,188 @@ trait JavaClientTemplateApi extends TemplateApi with JavaClientHelpers { } } + +// ==================== ENRICH POLICY API IMPLEMENTATION FOR JAVA CLIENT ==================== + +trait JavaClientEnrichPolicyApi extends EnrichPolicyApi with JavaClientHelpers { + _: JavaClientVersionApi with JavaClientCompanion => + + override private[client] def executeCreateEnrichPolicy( + policy: EnrichPolicy + ): ElasticResult[Boolean] = { + implicit val timestamp: Long = System.currentTimeMillis() + executeJavaBooleanAction( + operation = "createEnrichPolicy", + index = None, + retryable = false + )( + apply() + .enrich() + .putPolicy( + new PutPolicyRequest.Builder() + .name(policy.name) + .withJson(new StringReader(policy.node)) + .build() + ) + )(resp => resp.acknowledged()) + } + + override private[client] def executeDeleteEnrichPolicy( + policyName: String + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "deleteEnrichPolicy", + index = None, + retryable = false + )( + apply() + .enrich() + .deletePolicy( + new DeletePolicyRequest.Builder() + .name(policyName) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeExecuteEnrichPolicy( + policyName: JSONQuery + ): ElasticResult[JSONQuery] = + executeJavaAction( + operation = "executeEnrichPolicy", + index = None, + retryable = false + )( + apply() + .enrich() + .executePolicy( + new ExecutePolicyRequest.Builder() + .name(policyName) + .waitForCompletion(true) + .build() + ) + )(resp => resp.task()) +} + +// ==================== TRANSFORM API IMPLEMENTATION FOR JAVA CLIENT ==================== + +trait JavaClientTransformApi extends TransformApi with JavaClientHelpers { + _: JavaClientVersionApi with JavaClientCompanion => + + override private[client] def executeCreateTransform( + config: schema.TransformConfig, + start: Boolean + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "createTransform", + index = None, + retryable = false + ) { + implicit val timestamp: Long = System.currentTimeMillis() + implicit val context: PainlessContextType = PainlessContextType.Transform + apply() + .transform() + .putTransform( + new PutTransformRequest.Builder() + .transformId(config.id) + .withJson(new StringReader(convertToElasticTransformConfig(config))) + .build() + ) + }(resp => resp.acknowledged()) + + private def convertToElasticTransformConfig( + config: schema.TransformConfig + )(implicit criteriaToNode: Criteria => JsonNode): String = { + config.node + } + + override private[client] def executeDeleteTransform( + transformId: String, + force: Boolean + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "deleteTransform", + index = None, + retryable = false + )( + apply() + .transform() + .deleteTransform( + new DeleteTransformRequest.Builder() + .transformId(transformId) + .force(force) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeStartTransform(transformId: String): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "startTransform", + index = None, + retryable = false + )( + apply() + .transform() + .startTransform( + new StartTransformRequest.Builder() + .transformId(transformId) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeStopTransform( + transformId: String, + force: Boolean, + waitForCompletion: Boolean + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "stopTransform", + index = None, + retryable = false + )( + apply() + .transform() + .stopTransform( + new StopTransformRequest.Builder() + .transformId(transformId) + .force(force) + .waitForCompletion(waitForCompletion) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeGetTransformStats( + transformId: String + ): ElasticResult[Option[schema.TransformStats]] = + executeJavaAction( + operation = "getTransformStats", + index = None, + retryable = true + )( + apply() + .transform() + .getTransformStats( + new GetTransformStatsRequest.Builder() + .transformId(transformId) + .build() + ) + ) { resp => + val statsOpt = resp.transforms().asScala.headOption.map { stats => + schema.TransformStats( + id = stats.id(), + state = TransformState(stats.state()), + documentsProcessed = stats.stats().documentsProcessed(), + documentsIndexed = stats.stats().documentsIndexed(), + indexFailures = stats.stats().indexFailures(), + searchFailures = stats.stats().searchFailures(), + lastCheckpoint = Option(stats.checkpointing()) + .flatMap(c => Option(c.last())) + .map(_.checkpoint()), + operationsBehind = Option(stats.checkpointing()) + .flatMap(info => Option(info.operationsBehind().longValue())) + .getOrElse(0L), + processingTimeMs = stats.stats().processingTimeInMs() + ) + } + statsOpt + } +} diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 3a9a1eaa..3a612ece 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -24,15 +24,15 @@ import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} +import app.softnetwork.elastic.sql.query.{Criteria, SQLAggregation, SingleSearch} import app.softnetwork.elastic.client.result.{ ElasticError, ElasticFailure, ElasticResult, ElasticSuccess } -import app.softnetwork.elastic.sql.PainlessContextType -import app.softnetwork.elastic.sql.schema.TableAlias +import app.softnetwork.elastic.sql.{schema, PainlessContextType} +import app.softnetwork.elastic.sql.schema.{EnrichPolicy, TableAlias, TransformState} import app.softnetwork.elastic.sql.serialization._ import co.elastic.clients.elasticsearch._types.mapping.TypeMapping import co.elastic.clients.elasticsearch._types.{ @@ -56,6 +56,11 @@ import co.elastic.clients.elasticsearch.core.msearch.{MultisearchHeader, Request import co.elastic.clients.elasticsearch.core._ import co.elastic.clients.elasticsearch.core.reindex.{Destination, Source => ESSource} import co.elastic.clients.elasticsearch.core.search.{PointInTimeReference, SearchRequestBody} +import co.elastic.clients.elasticsearch.enrich.{ + DeletePolicyRequest, + ExecutePolicyRequest, + PutPolicyRequest +} import co.elastic.clients.elasticsearch.indices.update_aliases.{Action, AddAction, RemoveAction} import co.elastic.clients.elasticsearch.indices.{ExistsRequest => IndexExistsRequest, _} import co.elastic.clients.elasticsearch.ingest.{ @@ -63,6 +68,13 @@ import co.elastic.clients.elasticsearch.ingest.{ GetPipelineRequest, PutPipelineRequest } +import co.elastic.clients.elasticsearch.transform.{ + DeleteTransformRequest, + GetTransformStatsRequest, + PutTransformRequest, + StartTransformRequest, + StopTransformRequest +} import com.fasterxml.jackson.databind.JsonNode import com.google.gson.JsonParser @@ -93,6 +105,8 @@ trait JavaClientApi with JavaClientVersionApi with JavaClientPipelineApi with JavaClientTemplateApi + with JavaClientEnrichPolicyApi + with JavaClientTransformApi /** Elasticsearch client implementation using the Java Client * @see @@ -559,6 +573,26 @@ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { } } + override private[client] def executeGetAllMappings(): ElasticResult[Map[String, String]] = + executeJavaAction( + operation = "getAllMappings", + index = None, + retryable = true + )( + apply() + .indices() + .getMapping( + new GetMappingRequest.Builder().build() + ) + ) { response => + response + .mappings() + .asScala + .map { case (index, mapping) => + (index, convertToJson(mapping)) + } + .toMap + } } /** Elasticsearch client implementation of Refresh API using the Java Client @@ -1949,3 +1983,188 @@ trait JavaClientTemplateApi extends TemplateApi with JavaClientHelpers with Java } } + +// ==================== ENRICH POLICY API IMPLEMENTATION FOR JAVA CLIENT ==================== + +trait JavaClientEnrichPolicyApi extends EnrichPolicyApi with JavaClientHelpers { + _: JavaClientVersionApi with JavaClientCompanion => + + override private[client] def executeCreateEnrichPolicy( + policy: EnrichPolicy + ): ElasticResult[Boolean] = { + implicit val timestamp: Long = System.currentTimeMillis() + executeJavaBooleanAction( + operation = "createEnrichPolicy", + index = None, + retryable = false + )( + apply() + .enrich() + .putPolicy( + new PutPolicyRequest.Builder() + .name(policy.name) + .withJson(new StringReader(policy.node)) + .build() + ) + )(resp => resp.acknowledged()) + } + + override private[client] def executeDeleteEnrichPolicy( + policyName: String + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "deleteEnrichPolicy", + index = None, + retryable = false + )( + apply() + .enrich() + .deletePolicy( + new DeletePolicyRequest.Builder() + .name(policyName) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeExecuteEnrichPolicy( + policyName: String + ): ElasticResult[String] = + executeJavaAction( + operation = "executeEnrichPolicy", + index = None, + retryable = false + )( + apply() + .enrich() + .executePolicy( + new ExecutePolicyRequest.Builder() + .name(policyName) + .waitForCompletion(true) + .build() + ) + )(resp => resp.task()) +} + +// ==================== TRANSFORM API IMPLEMENTATION FOR JAVA CLIENT ==================== + +trait JavaClientTransformApi extends TransformApi with JavaClientHelpers { + _: JavaClientVersionApi with JavaClientCompanion => + + override private[client] def executeCreateTransform( + config: schema.TransformConfig, + start: Boolean + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "createTransform", + index = None, + retryable = false + ) { + implicit val timestamp: Long = System.currentTimeMillis() + implicit val context: PainlessContextType = PainlessContextType.Transform + apply() + .transform() + .putTransform( + new PutTransformRequest.Builder() + .transformId(config.id) + .withJson(new StringReader(convertToElasticTransformConfig(config))) + .build() + ) + }(resp => resp.acknowledged()) + + private def convertToElasticTransformConfig( + config: schema.TransformConfig + )(implicit criteriaToNode: Criteria => JsonNode): String = { + config.node + } + + override private[client] def executeDeleteTransform( + transformId: String, + force: Boolean + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "deleteTransform", + index = None, + retryable = false + )( + apply() + .transform() + .deleteTransform( + new DeleteTransformRequest.Builder() + .transformId(transformId) + .force(force) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeStartTransform(transformId: String): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "startTransform", + index = None, + retryable = false + )( + apply() + .transform() + .startTransform( + new StartTransformRequest.Builder() + .transformId(transformId) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeStopTransform( + transformId: String, + force: Boolean, + waitForCompletion: Boolean + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "stopTransform", + index = None, + retryable = false + )( + apply() + .transform() + .stopTransform( + new StopTransformRequest.Builder() + .transformId(transformId) + .force(force) + .waitForCompletion(waitForCompletion) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeGetTransformStats( + transformId: String + ): ElasticResult[Option[schema.TransformStats]] = + executeJavaAction( + operation = "getTransformStats", + index = None, + retryable = true + )( + apply() + .transform() + .getTransformStats( + new GetTransformStatsRequest.Builder() + .transformId(transformId) + .build() + ) + ) { resp => + val statsOpt = resp.transforms().asScala.headOption.map { stats => + schema.TransformStats( + id = stats.id(), + state = TransformState(stats.state()), + documentsProcessed = stats.stats().documentsProcessed(), + documentsIndexed = stats.stats().documentsIndexed(), + indexFailures = stats.stats().indexFailures(), + searchFailures = stats.stats().searchFailures(), + lastCheckpoint = Option(stats.checkpointing()) + .flatMap(c => Option(c.last())) + .map(_.checkpoint()), + operationsBehind = Option(stats.checkpointing()) + .flatMap(info => Option(info.operationsBehind().longValue())) + .getOrElse(0L), + processingTimeMs = stats.stats().processingTimeInMs() + ) + } + statsOpt + } +} diff --git a/licensing/src/main/scala/app/softnetwork/elastic/licensing/package.scala b/licensing/src/main/scala/app/softnetwork/elastic/licensing/package.scala index 98175b28..e67c2caa 100644 --- a/licensing/src/main/scala/app/softnetwork/elastic/licensing/package.scala +++ b/licensing/src/main/scala/app/softnetwork/elastic/licensing/package.scala @@ -18,12 +18,21 @@ package app.softnetwork.elastic package object licensing { - sealed trait LicenseType + sealed trait LicenseType { + def isPaid: Boolean = this != LicenseType.Community + def isEnterprise: Boolean = this == LicenseType.Enterprise + def isPro: Boolean = this == LicenseType.Pro + } object LicenseType { case object Community extends LicenseType // Gratuit case object Pro extends LicenseType // Payant case object Enterprise extends LicenseType // Payant + support + def upgradeTo(licenseType: LicenseType): LicenseType = licenseType match { + case Community => Pro + case Pro => Enterprise + case Enterprise => Enterprise + } } sealed trait Feature diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index 2ec3ade3..3501fbaf 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -237,6 +237,11 @@ package object schema { ) object IndexMappings { + def apply(json: String): IndexMappings = { + val root: JsonNode = json + apply(root) + } + def apply(root: JsonNode): IndexMappings = { if (root.has("mappings")) { val mappingsNode = root.path("mappings") diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index cf2bfb74..6af8df08 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -175,7 +175,7 @@ object Parser } def describePipeline: PackratParser[DescribePipeline] = - ("DESCRIBE" ~ "PIPELINE") ~ ident ^^ { case _ ~ pipeline => + (("DESCRIBE" | "DESC") ~ "PIPELINE") ~ ident ^^ { case _ ~ pipeline => DescribePipeline(pipeline) } @@ -364,6 +364,11 @@ object Parser } } + def showTables: PackratParser[ShowTables.type] = + ("SHOW" ~ "TABLES") ^^ { _ => + ShowTables + } + def showTable: PackratParser[ShowTable] = ("SHOW" ~ "TABLE") ~ ident ^^ { case _ ~ table => ShowTable(table) @@ -375,7 +380,7 @@ object Parser } def describeTable: PackratParser[DescribeTable] = - ("DESCRIBE" ~ "TABLE") ~ ident ^^ { case _ ~ table => + (("DESCRIBE" | "DESC") ~ "TABLE") ~ ident ^^ { case _ ~ table => DescribeTable(table) } @@ -401,6 +406,31 @@ object Parser CreateMaterializedView(view, dql, ifNotExists = ine, orReplace = false) } + def dropMaterializedView: PackratParser[DropMaterializedView] = + ("DROP" ~ "MATERIALIZED" ~ "VIEW") ~ ifExists ~ ident ^^ { case _ ~ ie ~ name => + DropMaterializedView(name, ifExists = ie) + } + + def showCreateMaterializedView: PackratParser[ShowCreateMaterializedView] = + ("SHOW" ~ "CREATE" ~ "MATERIALIZED" ~ "VIEW") ~ ident ^^ { case _ ~ _ ~ _ ~ _ ~ view => + ShowCreateMaterializedView(view) + } + + def showMaterializedView: PackratParser[ShowMaterializedView] = + ("SHOW" ~ "MATERIALIZED" ~ "VIEW") ~ ident ^^ { case _ ~ _ ~ view => + ShowMaterializedView(view) + } + + def showMaterializedViews: PackratParser[ShowMaterializedViews.type] = + ("SHOW" ~ "MATERIALIZED" ~ "VIEWS") ^^ { _ => + ShowMaterializedViews + } + + def describeMaterializedView: PackratParser[DescribeMaterializedView] = + (("DESCRIBE" | "DESC") ~ "MATERIALIZED" ~ "VIEW") ~ ident ^^ { case _ ~ _ ~ _ ~ view => + DescribeMaterializedView(view) + } + def addColumn: PackratParser[AddColumn] = ("ADD" ~ "COLUMN") ~ ifNotExists ~ column ^^ { case _ ~ ine ~ col => AddColumn(col, ifNotExists = ine) @@ -594,6 +624,7 @@ object Parser alterPipeline | dropTable | truncateTable | + showTables | showTable | showCreateTable | describeTable | @@ -602,7 +633,12 @@ object Parser showCreatePipeline | describePipeline | createMaterializedView | - createOrReplaceMaterializedView + createOrReplaceMaterializedView | + dropMaterializedView | + showMaterializedViews | + showMaterializedView | + showCreateMaterializedView | + describeMaterializedView def onConflict: PackratParser[OnConflict] = ("ON" ~ "CONFLICT" ~> opt(conflictTarget) <~ "DO") ~ ("UPDATE" | "NOTHING") ^^ { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala index d9beac93..38962d76 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala @@ -149,7 +149,7 @@ sealed trait Join extends Updateable { def on: Option[On] def alias: Option[Alias] override def sql: String = - s" ${asString(joinType)} $Join $source${asString(on)}${asString(alias)}" + s" ${asString(joinType)} $Join $source${asString(alias)}${asString(on)}" override def update(request: SingleSearch): Join diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index e71a4375..666acf68 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -565,7 +565,7 @@ package object query { case class CreateMaterializedView( view: String, - ddl: DqlStatement, + dql: DqlStatement, ifNotExists: Boolean = false, orReplace: Boolean = false, options: Map[String, Value[_]] = Map.empty @@ -573,16 +573,48 @@ package object query { override def sql: String = { val replaceClause = if (orReplace) " OR REPLACE" else "" val ineClause = if (!orReplace && ifNotExists) " IF NOT EXISTS" else "" - s"CREATE$replaceClause MATERIALIZED VIEW$ineClause $view AS ${ddl.sql}" + s"CREATE$replaceClause MATERIALIZED VIEW$ineClause $view AS ${dql.sql}" } - lazy val search: SingleSearch = ddl match { + lazy val search: SingleSearch = dql match { case s: SingleSearch => s case _ => throw new IllegalArgumentException("Materialized view must be a single search") } } + case class DropMaterializedView(name: String, ifExists: Boolean = false) + extends MaterializedViewStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) "IF EXISTS " else "" + s"DROP MATERIALIZED VIEW $ifExistsClause$name" + } + } + + case class RefreshMaterializedView(name: String, ifExists: Boolean = false) + extends MaterializedViewStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) "IF EXISTS " else "" + s"REFRESH MATERIALIZED VIEW $ifExistsClause$name" + } + } + + case class ShowMaterializedView(name: String) extends MaterializedViewStatement { + override def sql: String = s"SHOW MATERIALIZED VIEW $name" + } + + case object ShowMaterializedViews extends MaterializedViewStatement { + override def sql: String = s"SHOW MATERIALIZED VIEWS" + } + + case class ShowCreateMaterializedView(name: String) extends MaterializedViewStatement { + override def sql: String = s"SHOW CREATE MATERIALIZED VIEW $name" + } + + case class DescribeMaterializedView(name: String) extends MaterializedViewStatement { + override def sql: String = s"DESCRIBE MATERIALIZED VIEW $name" + } + case class CreateTable( table: String, ddl: Either[DqlStatement, List[Column]], @@ -936,6 +968,10 @@ package object query { override def sql: String = s"SHOW TABLE $table" } + case object ShowTables extends TableStatement { + override def sql: String = s"SHOW TABLES" + } + case class ShowCreateTable(table: String) extends TableStatement { override def sql: String = s"SHOW CREATE TABLE $table" } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index e905af66..fd200d01 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -30,7 +30,7 @@ import app.softnetwork.elastic.sql.query._ import app.softnetwork.elastic.sql.serialization._ import app.softnetwork.elastic.sql.time.TimeUnit import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} -import com.fasterxml.jackson.databind.node.ObjectNode +import com.fasterxml.jackson.databind.node.{NullNode, ObjectNode} import java.util.UUID import scala.jdk.CollectionConverters._ @@ -678,6 +678,8 @@ package object schema { ) } } + + def describe: Seq[Map[String, Any]] = processors.map(_.properties) } object IngestPipeline { @@ -1127,6 +1129,58 @@ package object schema { override def sql: String = s"COUNT(DISTINCT $field)" } + case class TopHitsTransformAggregation( + fields: Seq[String], + size: Int = 1, + sort: Seq[FieldSort] = Seq( + FieldSort(Identifier(sqlConfig.transformLastUpdatedColumnName), Some(Desc)) + ) + ) extends TransformAggregation { + override def field: String = fields.mkString(", ") + + override def name: String = "top_hits" + + override def sql: String = { + val fieldsStr = fields.mkString(", ") + val sortStr = sort + .map { s => + s.sql + } + .mkString(", ") + s"TOP_HITS($fieldsStr) SIZE $size SORT BY ($sortStr)" + } + + override def node: JsonNode = { + val node = mapper.createObjectNode() + val topHitsNode = mapper.createObjectNode() + topHitsNode.put("size", size) + + // Sort configuration + val sortArray = mapper.createArrayNode() + sort.foreach { sortField => + val sortObj = mapper.createObjectNode() + val sortFieldObj = mapper.createObjectNode() + sortFieldObj.put("order", sortField.order.getOrElse(Desc).sql) + sortObj.set(sortField.name, sortFieldObj) + sortArray.add(sortObj) + } + topHitsNode.set("sort", sortArray) + + // Source filtering to include only the desired field + val sourceObj = mapper.createObjectNode() + val includesArray = mapper.createArrayNode() + fields.foreach { field => + includesArray.add(field) + () + } + sourceObj.set("includes", includesArray) + topHitsNode.set("_source", sourceObj) + + node.set("top_hits", topHitsNode) + node + } + } + /** Extension methods for AggregateFunction */ implicit class AggregateConversion(agg: AggregateFunction) { @@ -1162,69 +1216,6 @@ package object schema { } } - /** Captures the latest value for a field (for changelog snapshots) - * - * Uses top_hits with size=1 sorted by a timestamp field This works for ANY field type (text, - * numeric, date, etc.) - */ - case class LatestValueTransformAggregation( - field: String, - sortBy: String = "_ingest.timestamp" // Default sort by ingest timestamp - ) extends TransformAggregation { - - override def name: String = "latest_value" - - override def sql: String = s"LAST_VALUE($field)" - - override def node: JsonNode = { - val node = mapper.createObjectNode() - val topHitsNode = mapper.createObjectNode() - - // Configure top_hits to get the latest value - topHitsNode.put("size", 1) - - // Sort by timestamp descending to get latest - val sortArray = mapper.createArrayNode() - val sortObj = mapper.createObjectNode() - val sortFieldObj = mapper.createObjectNode() - sortFieldObj.put("order", "desc") - sortObj.set(sortBy, sortFieldObj) - sortArray.add(sortObj) - topHitsNode.set("sort", sortArray) - - // Only retrieve the field we need - val sourceObj = mapper.createObjectNode() - val includesArray = mapper.createArrayNode() - includesArray.add(field) - sourceObj.set("includes", includesArray) - topHitsNode.set("_source", sourceObj) - - node.set("top_hits", topHitsNode) - node - } - } - - /** Alternative: Use MAX aggregation for compatible types - * - * This is simpler but only works for: - * - Numeric fields (int, long, double, float) - * - Date fields - * - Keyword fields (lexicographic max) - */ - case class MaxValueTransformAggregation(field: String) extends TransformAggregation { - override def name: String = "max" - - override def sql: String = s"MAX($field)" - } - - /** Alternative: Use MIN aggregation (for specific use cases) - */ - case class MinValueTransformAggregation(field: String) extends TransformAggregation { - override def name: String = "min" - - override def sql: String = s"MIN($field)" - } - object ChangelogAggregationStrategy { /** Selects the appropriate aggregation for a field in a changelog based on its data type @@ -1240,27 +1231,27 @@ package object schema { dataType match { // Numeric types: Use MAX directly case SQLTypes.Int | SQLTypes.BigInt | SQLTypes.Double | SQLTypes.Real => - MaxValueTransformAggregation(field) + MaxTransformAggregation(field) // Date/Timestamp: Use MAX directly case SQLTypes.Date | SQLTypes.Timestamp => - MaxValueTransformAggregation(field) + MaxTransformAggregation(field) - // Boolean: Use MAX directly (1=true, 0=false) + // Boolean: Use Top Hits case SQLTypes.Boolean => - MaxValueTransformAggregation(field) + TopHitsTransformAggregation(Seq(field)) // Keyword: Already a keyword type, use MAX directly case SQLTypes.Keyword => - MaxValueTransformAggregation(field) // ✅ NO .keyword suffix + TopHitsTransformAggregation(Seq(field)) // Text/Varchar: Analyzed field, must use .keyword multi-field case SQLTypes.Text | SQLTypes.Varchar => - MaxValueTransformAggregation(s"$field.keyword") // ✅ Use .keyword multi-field + TopHitsTransformAggregation(Seq(s"$field.keyword")) // ✅ Use .keyword multi-field // For complex types, use top_hits as fallback case _ => - LatestValueTransformAggregation(field) + TopHitsTransformAggregation(Seq(field)) } } @@ -1302,16 +1293,46 @@ package object schema { } } + case class TransformCreationStatus( + created: Boolean, + started: Option[Boolean] = None // None = not asked, Some(true/false) = result of start request + ) + + case class TransformLatest( + uniqueKey: Seq[String], + sort: String + ) extends DdlToken { + override def sql: String = { + val uniqueKeyStr = uniqueKey.mkString(", ") + s"LATEST UNIQUE KEY ($uniqueKeyStr) SORT BY $sort" + } + + def node: JsonNode = { + val node = mapper.createObjectNode() + val uniqueKeyNode = mapper.createArrayNode() + uniqueKey.foreach { key => + uniqueKeyNode.add(key) + () + } + node.set("unique_key", uniqueKeyNode) + node.put("sort", sort) + node + } + } + /** Transform configuration models with built-in JSON serialization */ case class TransformConfig( id: String, + viewName: String, source: TransformSource, dest: TransformDest, pivot: Option[TransformPivot] = None, + latest: Option[TransformLatest] = None, sync: Option[TransformSync] = None, delay: Delay, - frequency: Frequency + frequency: Frequency, + metadata: Map[String, AnyRef] = Map.empty ) extends DdlToken { /** Delay in seconds for display */ @@ -1335,6 +1356,10 @@ package object schema { node.set("pivot", p.node) () } + latest.foreach { l => + node.set("latest", l.node) + () + } node } @@ -1361,14 +1386,115 @@ package object schema { sb.toString() } - /** Human-readable summary + /** Human-readable description */ - def summary: String = { + def description: String = { s"Transform $id: ${source.index.mkString(", ")} → ${dest.index} " + - s"(every ${frequencySeconds}s, delay ${delaySeconds}s)" + s"(every ${frequencySeconds}s, delay ${delaySeconds}s) for view $viewName" + } + } + + /** Health status + */ + sealed trait HealthStatus extends DdlToken { + def name: String + def sql: String = name + } + + object HealthStatus { + case object Green extends HealthStatus { + val name: String = "GREEN" + } + case object Yellow extends HealthStatus { + val name: String = "YELLOW" + } + case object Red extends HealthStatus { + val name: String = "RED" + } + + def apply(name: String): HealthStatus = name.toUpperCase() match { + case "GREEN" => Green + case "YELLOW" => Yellow + case "RED" => Red + case other => throw new IllegalArgumentException(s"Invalid health status: $other") + } + } + + /** Transform state + */ + sealed trait TransformState extends DdlToken { + def name: String + def sql: String = name + } + + object TransformState { + case object Started extends TransformState { + val name: String = "STARTED" + } + case object Indexing extends TransformState { + val name: String = "INDEXING" + } + case object Aborting extends TransformState { + val name: String = "ABORTING" + } + case object Stopping extends TransformState { + val name: String = "STOPPING" + } + case object Stopped extends TransformState { + val name: String = "STOPPED" + } + case object Failed extends TransformState { + val name: String = "FAILED" + } + case object Waiting extends TransformState { + val name: String = "WAITING" + } + case class Other(name: String) extends TransformState + + def apply(name: String): TransformState = name.toUpperCase() match { + case "STARTED" => Started + case "INDEXING" => Indexing + case "ABORTING" => Aborting + case "STOPPING" => Stopping + case "STOPPED" => Stopped + case "FAILED" => Failed + case "WAITING" => Waiting + case other => Other(other) } } + /** Statistics for a Transform + * + * @param id + * Transform ID + * @param state + * Current state of the Transform + * @param documentsProcessed + * Total number of documents processed + * @param documentsIndexed + * Total number of documents indexed + * @param indexFailures + * Total number of indexing failures + * @param searchFailures + * Total number of search failures + * @param lastCheckpoint + * Last checkpoint number (if any) + * @param operationsBehind + * Number of operations behind source index + * @param processingTimeMs + * Total processing time in milliseconds + */ + case class TransformStats( + id: String, + state: TransformState, // "started", "stopped", "failed" + documentsProcessed: Long, + documentsIndexed: Long, + indexFailures: Long, + searchFailures: Long, + lastCheckpoint: Option[Long], + operationsBehind: Long, + processingTimeMs: Long + ) // ==================== Schema ==================== /** Definition of a column within a table @@ -1993,6 +2119,12 @@ package object schema { ) } + lazy val metadata: Map[String, Any] = + mappings.get("_meta") match { + case Some(ObjectValue(value)) => ObjectValue(value) + case _ => Map.empty + } + def sql: String = { val opts = if (mappings.nonEmpty || settings.nonEmpty) { @@ -2505,6 +2637,8 @@ package object schema { aliases = aliasDiffs.toList ) } + + def describe: Seq[Map[String, Any]] = columns.flatMap(_.asMap) } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala index 24a1df33..ea9775bf 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala @@ -90,6 +90,12 @@ package object serialization { node } + implicit def objectValueToMap(value: ObjectValue): Map[String, Any] = { + import JacksonConfig.{objectMapper => mapper} + val node: ObjectNode = value + mapper.convertValue(node, classOf[Map[String, Any]]) + } + private[sql] def updateNode(node: ObjectNode, updates: Map[String, Value[_]]): ObjectNode = { import JacksonConfig.{objectMapper => mapper} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala index ce8fa93b..c48cf40e 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala @@ -93,7 +93,8 @@ trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala def assertSelectResult( res: ElasticResult[QueryResult], - rows: Seq[Map[String, Any]] = Seq.empty + rows: Seq[Map[String, Any]] = Seq.empty, + nbResults: Option[Int] = None ): Unit = { res.isSuccess shouldBe true res.toOption.get match { @@ -106,6 +107,8 @@ trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala if (rows.nonEmpty) { results.size shouldBe rows.size results should contain theSameElementsAs rows + } else if (nbResults.isDefined) { + results.size shouldBe nbResults.get } else { log.info(s"Rows: $results") } @@ -115,6 +118,8 @@ trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala if (rows.nonEmpty) { results.size shouldBe rows.size results should contain theSameElementsAs rows + } else if (nbResults.isDefined) { + results.size shouldBe nbResults.get } else { log.info(s"Rows: $results") } @@ -123,6 +128,8 @@ trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala if (rows.nonEmpty) { results.size shouldBe rows.size results should contain theSameElementsAs rows + } else if (nbResults.isDefined) { + results.size shouldBe nbResults.get } else { log.info(s"Rows: $results") } @@ -156,6 +163,16 @@ trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala } } + // ------------------------------------------------------------------------- + // Helper: assert SHOW TABLES result type + // ------------------------------------------------------------------------- + + def assertShowTables(res: ElasticResult[QueryResult]): Seq[Map[String, Any]] = { + res.isSuccess shouldBe true + res.toOption.get shouldBe a[QueryRows] + res.toOption.get.asInstanceOf[QueryRows].rows + } + // ------------------------------------------------------------------------- // Helper: assert SHOW TABLE result type // ------------------------------------------------------------------------- @@ -638,6 +655,17 @@ trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala table.ddl should include("visibility BOOLEAN DEFAULT true") } + it should "list all tables" in { + val tables = assertShowTables(client.run("SHOW TABLES").futureValue) + tables should not be empty + for { + table <- tables + } { + table should contain key "name" + table should contain key "type" + } + } + // =========================================================================== // 3. DML — INSERT / UPDATE / DELETE // =========================================================================== From 3914f46961041d9980cd52534de4d775c8bf3581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 23 Jan 2026 07:35:06 +0100 Subject: [PATCH 22/22] add support for refresh materialized view ddl, add support for show materialized view status ddl, update materialized view craetion ddl adding refresh and options --- .../elastic/sql/parser/Parser.scala | 58 ++++++++++++++++--- .../elastic/sql/query/package.scala | 51 +++++++++++++++- .../elastic/sql/schema/package.scala | 52 ++++++++++++----- 3 files changed, 136 insertions(+), 25 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 6af8df08..e5f951fc 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -32,11 +32,13 @@ import app.softnetwork.elastic.sql.parser.operator.math.ArithmeticParser import app.softnetwork.elastic.sql.query._ import app.softnetwork.elastic.sql.schema.{ Column, + Frequency, IngestPipelineType, IngestProcessor, IngestProcessorType, PartitionDate, - ScriptProcessor + ScriptProcessor, + TransformTimeUnit } import app.softnetwork.elastic.sql.time.TimeUnit @@ -394,16 +396,46 @@ object Parser TruncateTable(name) } + def frequency: PackratParser[Frequency] = + ("REFRESH" ~ "EVERY") ~> """\d+\s+(MILLISECOND|SECOND|MINUTE|HOUR|DAY|WEEK|MONTH|YEAR)S?""".r ^^ { + str => + val parts = str.trim.split("\\s+") + Frequency(TransformTimeUnit(parts(1)), parts(0).toInt) + } + + def withOptions: PackratParser[Map[String, Value[_]]] = + ("WITH" ~ lparen) ~> repsep(option, separator) <~ rparen ^^ { opts => + opts.toMap + } + def createOrReplaceMaterializedView: PackratParser[CreateMaterializedView] = - ("CREATE" ~ "OR" ~ "REPLACE" ~ "MATERIALIZED" ~ "VIEW") ~ ident ~ ("AS" ~> dqlStatement) ^^ { - case _ ~ view ~ dql => - CreateMaterializedView(view, dql, ifNotExists = false, orReplace = true) + ("CREATE" ~ "OR" ~ "REPLACE" ~ "MATERIALIZED" ~ "VIEW") ~ ident ~ opt(frequency) ~ opt( + withOptions + ) ~ ("AS" ~> dqlStatement) ^^ { case _ ~ view ~ freq ~ opts ~ dql => + CreateMaterializedView( + view, + dql, + ifNotExists = false, + orReplace = true, + frequency = freq, + options = opts.getOrElse(Map.empty) + ) } def createMaterializedView: PackratParser[CreateMaterializedView] = - ("CREATE" ~ "MATERIALIZED" ~ "VIEW") ~ ifNotExists ~ ident ~ ("AS" ~> dqlStatement) ^^ { - case _ ~ ine ~ view ~ dql => - CreateMaterializedView(view, dql, ifNotExists = ine, orReplace = false) + ("CREATE" ~ "MATERIALIZED" ~ "VIEW") ~ ifNotExists ~ ident ~ opt( + frequency + ) ~ opt( + withOptions + ) ~ ("AS" ~> dqlStatement) ^^ { case _ ~ ine ~ view ~ freq ~ opts ~ dql => + CreateMaterializedView( + view, + dql, + ifNotExists = ine, + orReplace = false, + frequency = freq, + options = opts.getOrElse(Map.empty) + ) } def dropMaterializedView: PackratParser[DropMaterializedView] = @@ -411,6 +443,16 @@ object Parser DropMaterializedView(name, ifExists = ie) } + def refreshMaterializedView: PackratParser[RefreshMaterializedView] = + ("REFRESH" ~ "MATERIALIZED" ~ "VIEW") ~ ident ^^ { case _ ~ _ ~ view => + RefreshMaterializedView(view) + } + + def showMaterializedViewStatus: PackratParser[ShowMaterializedViewStatus] = + ("SHOW" ~ "MATERIALIZED" ~ "VIEW" ~ "STATUS") ~ ident ^^ { case _ ~ _ ~ _ ~ _ ~ view => + ShowMaterializedViewStatus(view) + } + def showCreateMaterializedView: PackratParser[ShowCreateMaterializedView] = ("SHOW" ~ "CREATE" ~ "MATERIALIZED" ~ "VIEW") ~ ident ^^ { case _ ~ _ ~ _ ~ _ ~ view => ShowCreateMaterializedView(view) @@ -635,6 +677,8 @@ object Parser createMaterializedView | createOrReplaceMaterializedView | dropMaterializedView | + refreshMaterializedView | + showMaterializedViewStatus | showMaterializedViews | showMaterializedView | showCreateMaterializedView | diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index 666acf68..cf6d9d3d 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -20,6 +20,7 @@ import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} import app.softnetwork.elastic.sql.schema.{ sqlConfig, Column, + Frequency, IngestPipeline, IngestPipelineType, IngestProcessor, @@ -30,13 +31,14 @@ import app.softnetwork.elastic.sql.schema.{ ScriptProcessor, SetProcessor, Table => Schema, - TableType + TableType, + TransformTimeUnit } import app.softnetwork.elastic.sql.function.aggregate.WindowFunction import app.softnetwork.elastic.sql.serialization._ import com.fasterxml.jackson.databind.JsonNode -import java.time.Instant +import java.time.{Duration, Instant} package object query { sealed trait Statement extends Token @@ -568,12 +570,24 @@ package object query { dql: DqlStatement, ifNotExists: Boolean = false, orReplace: Boolean = false, + frequency: Option[Frequency] = None, options: Map[String, Value[_]] = Map.empty ) extends MaterializedViewStatement { override def sql: String = { + val frequencySql = frequency match { + case Some(freq) => freq.sql + case None => "" + } + val optionsSql = if (options.nonEmpty) { + s" WITH (${options + .map { case (k, v) => s"$k = $v" } + .mkString(", ")})" + } else { + "" + } val replaceClause = if (orReplace) " OR REPLACE" else "" val ineClause = if (!orReplace && ifNotExists) " IF NOT EXISTS" else "" - s"CREATE$replaceClause MATERIALIZED VIEW$ineClause $view AS ${dql.sql}" + s"CREATE$replaceClause MATERIALIZED VIEW$ineClause $view$frequencySql$optionsSql AS ${dql.sql}" } lazy val search: SingleSearch = dql match { @@ -581,6 +595,33 @@ package object query { case _ => throw new IllegalArgumentException("Materialized view must be a single search") } + lazy val userLatency: Duration = options.get("user_latency") match { + case Some(value) => + value match { + case s: StringValue => + val regex = """\d+\s+(ms|s|m|h|d|w|M|y)""".r + regex.findFirstIn(s.value) match { + case Some(str) => + val parts = str.trim.split("\\s+") + val unit = parts(1) match { + case "ms" => TransformTimeUnit.Milliseconds + case "s" => TransformTimeUnit.Seconds + case "m" => TransformTimeUnit.Minutes + case "h" => TransformTimeUnit.Hours + case "d" => TransformTimeUnit.Days + case "w" => TransformTimeUnit.Weeks + case "M" => TransformTimeUnit.Months + case "y" => TransformTimeUnit.Years + case _ => TransformTimeUnit.Seconds + } + Duration.ofSeconds(Frequency(unit, parts(0).toInt).toSeconds) + case None => Duration.ofSeconds(5) + } + case i: LongValue => Duration.ofSeconds(i.value) + case _ => Duration.ofSeconds(5) + } + case None => Duration.ofSeconds(5) + } } case class DropMaterializedView(name: String, ifExists: Boolean = false) @@ -599,6 +640,10 @@ package object query { } } + case class ShowMaterializedViewStatus(name: String) extends MaterializedViewStatement { + override def sql: String = s"SHOW MATERIALIZED VIEW STATUS $name" + } + case class ShowMaterializedView(name: String) extends MaterializedViewStatement { override def sql: String = s"SHOW MATERIALIZED VIEW $name" } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index fd200d01..e9b21fa1 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -763,47 +763,61 @@ package object schema { sealed trait TransformTimeUnit extends DdlToken { def name: String + def sql: String = name + def format: String } object TransformTimeUnit { case object Milliseconds extends TransformTimeUnit { val name: String = "MILLISECONDS" - override def sql: String = "ms" + override def format: String = "ms" } case object Seconds extends TransformTimeUnit { val name: String = "SECONDS" - override def sql: String = "s" + override def format: String = "s" } case object Minutes extends TransformTimeUnit { val name: String = "MINUTES" - override def sql: String = "m" + override def format: String = "m" } case object Hours extends TransformTimeUnit { val name: String = "HOURS" - override def sql: String = "h" + override def format: String = "h" } case object Days extends TransformTimeUnit { val name: String = "DAYS" - override def sql: String = "d" + override def format: String = "d" } case object Weeks extends TransformTimeUnit { val name: String = "WEEKS" - override def sql: String = "w" + override def format: String = "w" + } + + case object Months extends TransformTimeUnit { + val name: String = "MONTHS" + override def format: String = "M" + } + + case object Years extends TransformTimeUnit { + val name: String = "YEARS" + override def format: String = "y" } def apply(name: String): TransformTimeUnit = name.toUpperCase() match { - case "MILLISECONDS" => Milliseconds - case "SECONDS" => Seconds - case "MINUTES" => Minutes - case "HOURS" => Hours - case "DAYS" => Days - case "WEEKS" => Weeks - case other => throw new IllegalArgumentException(s"Invalid delay unit: $other") + case "MILLISECOND" | "MILLISECONDS" => Milliseconds + case "SECOND" | "SECONDS" => Seconds + case "MINUTE" | "MINUTES" => Minutes + case "HOUR" | "HOURS" => Hours + case "DAY" | "DAYS" => Days + case "WEEK" | "WEEKS" => Weeks + case "MONTH" | "MONTHS" => Months + case "YEAR" | "YEARS" => Years + case other => throw new IllegalArgumentException(s"Invalid delay unit: $other") } } @@ -817,8 +831,10 @@ package object schema { case TransformTimeUnit.Hours => interval * 3600 case TransformTimeUnit.Days => interval * 86400 case TransformTimeUnit.Weeks => interval * 604800 + case TransformTimeUnit.Months => interval * 2592000 + case TransformTimeUnit.Years => interval * 31536000 } - def toTransformFormat: String = s"$interval$timeUnit" + def toTransformFormat: String = s"$interval${timeUnit.format}" } @@ -826,7 +842,13 @@ package object schema { /** Creates a time interval from seconds */ def fromSeconds(seconds: Int): (TransformTimeUnit, Int) = { - if (seconds >= 86400 && seconds % 86400 == 0) { + if (seconds >= 31536000 && seconds % 31536000 == 0) { + (TransformTimeUnit.Years, seconds / 31536000) + } else if (seconds >= 2592000 && seconds % 2592000 == 0) { + (TransformTimeUnit.Months, seconds / 2592000) + } else if (seconds >= 604800 && seconds % 604800 == 0) { + (TransformTimeUnit.Weeks, seconds / 604800) + } else if (seconds >= 86400 && seconds % 86400 == 0) { (TransformTimeUnit.Days, seconds / 86400) } else if (seconds >= 3600 && seconds % 3600 == 0) { (TransformTimeUnit.Hours, seconds / 3600)