diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index 72132c8d..0c303fa1 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.PainlessContext +import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType} import app.softnetwork.elastic.sql.`type`.SQLTemporal import app.softnetwork.elastic.sql.query.{ Asc, @@ -100,7 +100,10 @@ object ElasticAggregation { having: Option[Criteria], bucketsDirection: Map[String, SortOrder], allAggregations: Map[String, SQLAggregation] - )(implicit timestamp: Long): ElasticAggregation = { + )(implicit + timestamp: Long, + contextType: PainlessContextType + ): ElasticAggregation = { import sqlAgg._ val sourceField = identifier.path @@ -153,14 +156,14 @@ object ElasticAggregation { buildScript: (String, Script) => Aggregation ): Aggregation = { if (transformFuncs.nonEmpty) { - val context = PainlessContext() + val context = PainlessContext(context = contextType) val scriptSrc = identifier.painless(Some(context)) val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) } else { aggType match { case th: WindowFunction if th.shouldBeScripted => - val context = PainlessContext() + val context = PainlessContext(context = contextType) val scriptSrc = th.identifier.painless(Some(context)) val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) @@ -348,7 +351,10 @@ object ElasticAggregation { having: Option[Criteria], nested: Option[NestedElement], allElasticAggregations: Seq[ElasticAggregation] - )(implicit timestamp: Long): Seq[Aggregation] = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Seq[Aggregation] = { for (tree <- buckets) yield { val treeNodes = tree.sortBy(_.level).reverse.foldLeft(Seq.empty[NodeAggregation]) { (current, node) => @@ -364,7 +370,7 @@ object ElasticAggregation { val aggScript = if (!bucket.isBucketScript && bucket.shouldBeScripted) { - val context = PainlessContext() + val context = PainlessContext(context = contextType) val painless = bucket.painless(Some(context)) Some(now(Script(s"$context$painless").lang("painless"))) } else { diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala index c013e82c..c3b2893f 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala @@ -16,6 +16,7 @@ package app.softnetwork.elastic.sql.bridge +import app.softnetwork.elastic.sql.PainlessContextType import app.softnetwork.elastic.sql.operator.AND import app.softnetwork.elastic.sql.query.{ BetweenExpr, @@ -36,16 +37,16 @@ import app.softnetwork.elastic.sql.query.{ Predicate } import com.sksamuel.elastic4s.ElasticApi._ -import com.sksamuel.elastic4s.requests.common.FetchSourceContext import com.sksamuel.elastic4s.requests.searches.queries.{InnerHit, Query} -import scala.annotation.tailrec - case class ElasticBridge(filter: ElasticFilter) { def query( innerHitsNames: Set[String] = Set.empty, currentQuery: Option[ElasticBoolQuery] - )(implicit timestamp: Long): Query = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Query = { filter match { case boolQuery: ElasticBoolQuery => import boolQuery._ diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala index 32305e18..89e86e96 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala @@ -16,13 +16,15 @@ package app.softnetwork.elastic.sql.bridge +import app.softnetwork.elastic.sql.PainlessContextType import app.softnetwork.elastic.sql.query.Criteria import com.sksamuel.elastic4s.requests.searches.queries.Query case class ElasticCriteria(criteria: Criteria) { def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty)(implicit - timestamp: Long + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query ): Query = { val query = criteria.boolQuery.copy(group = group) query diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index d8de91e1..36c73605 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -23,13 +23,16 @@ import app.softnetwork.elastic.sql.`type`.{ SQLTemporal, SQLVarchar } +import app.softnetwork.elastic.sql.config.ElasticSqlConfig import app.softnetwork.elastic.sql.function.aggregate.COUNT import app.softnetwork.elastic.sql.function.geo.{Distance, Meters} import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql.query._ +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.NullNode import com.sksamuel.elastic4s.ElasticApi import com.sksamuel.elastic4s.ElasticApi._ -import com.sksamuel.elastic4s.requests.common.FetchSourceContext +import com.sksamuel.elastic4s.json.JacksonBuilder import com.sksamuel.elastic4s.requests.script.Script import com.sksamuel.elastic4s.requests.script.ScriptType.Source import com.sksamuel.elastic4s.requests.searches.aggs.{ @@ -51,17 +54,30 @@ import scala.language.implicitConversions package object bridge { - def now(script: Script)(implicit timestamp: Long): Script = { + lazy val sqlConfig: ElasticSqlConfig = ElasticSqlConfig() + + def now(script: Script)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Script = { if (!script.script.contains("params.__now__")) { return script } - script.param("__now__", timestamp) + contextType match { + case PainlessContextType.Query => script.param("__now__", timestamp) + case PainlessContextType.Transform => + script.param("__now__", sqlConfig.transformLastUpdatedColumnName) + case _ => script + } } implicit def requestToNestedFilterAggregation( request: SingleSearch, innerHitsName: String - )(implicit timestamp: Long): Option[FilterAggregation] = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Option[FilterAggregation] = { val having: Option[Query] = request.having.flatMap(_.criteria) match { case Some(f) => @@ -137,7 +153,10 @@ package object bridge { implicit def requestToFilterAggregation( request: SingleSearch - )(implicit timestamp: Long): Option[FilterAggregation] = + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Option[FilterAggregation] = request.having.flatMap(_.criteria) match { case Some(f) => val boolQuery = Option(ElasticBoolQuery(group = true)) @@ -155,7 +174,10 @@ package object bridge { implicit def requestToRootAggregations( request: SingleSearch, aggregations: Seq[ElasticAggregation] - )(implicit timestamp: Long): Seq[AbstractAggregation] = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Seq[AbstractAggregation] = { val notNestedAggregations = aggregations.filterNot(_.nested) val notNestedBuckets = request.bucketTree.filterNot(_.bucket.nested) @@ -207,7 +229,10 @@ package object bridge { implicit def requestToScopedAggregations( request: SingleSearch, aggregations: Seq[ElasticAggregation] - )(implicit timestamp: Long): Seq[NestedAggregation] = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Seq[NestedAggregation] = { // Group nested aggregations by their nested path val nestedAggregations: Map[String, Seq[ElasticAggregation]] = aggregations .filter(_.nested) @@ -413,7 +438,8 @@ package object bridge { } implicit def requestToElasticSearchRequest(request: SingleSearch)(implicit - timestamp: Long + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query ): ElasticSearchRequest = ElasticSearchRequest( request.sql, @@ -431,7 +457,10 @@ package object bridge { implicit def requestToSearchRequest( request: SingleSearch - )(implicit timestamp: Long): SearchRequest = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): SearchRequest = { import request._ val aggregations = request.aggregates.map( @@ -491,7 +520,7 @@ package object bridge { case Nil => _search case _ => _search scriptfields scriptFields.map { field => - val context = PainlessContext() + val context = PainlessContext(context = contextType) val script = field.painless(Some(context)) scriptField( field.scriptName, @@ -512,7 +541,7 @@ package object bridge { case Some(o) if aggregates.isEmpty && buckets.isEmpty => _search sortBy o.sorts.map { sort => if (sort.isScriptSort) { - val context = PainlessContext() + val context = PainlessContext(context = contextType) val painless = sort.field.painless(Some(context)) val painlessScript = s"$context$painless" val script = @@ -571,7 +600,10 @@ package object bridge { implicit def requestToMultiSearchRequest( request: MultiSearch - )(implicit timestamp: Long): MultiSearchRequest = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): MultiSearchRequest = { MultiSearchRequest( request.requests.map(implicitly[SearchRequest](_)) ) @@ -582,7 +614,10 @@ package object bridge { doubleOp: Double => A ): A = n.toEither.fold(longOp, doubleOp) - implicit def expressionToQuery(expression: GenericExpression)(implicit timestamp: Long): Query = { + implicit def expressionToQuery(expression: GenericExpression)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Query = { import expression._ if (isAggregation) return matchAllQuery() @@ -592,7 +627,7 @@ package object bridge { case _ => true })) ) { - val context = PainlessContext() + val context = PainlessContext(context = contextType) val script = painless(Some(context)) return scriptQuery( now(Script(script = s"$context$script").lang("painless").scriptType("source")) @@ -810,7 +845,7 @@ package object bridge { case NE | DIFF => not(rangeQuery(identifier.name) gte script lte script) } case _ => - val context = PainlessContext() + val context = PainlessContext(context = contextType) val script = painless(Some(context)) scriptQuery( now( @@ -821,7 +856,7 @@ package object bridge { ) } case _ => - val context = PainlessContext() + val context = PainlessContext(context = contextType) val script = painless(Some(context)) scriptQuery( now( @@ -884,7 +919,10 @@ package object bridge { implicit def betweenToQuery( between: BetweenExpr - )(implicit timestamp: Long): Query = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Query = { import between._ // Geo distance special case identifier.functions.headOption match { @@ -1007,6 +1045,40 @@ package object bridge { ) } + implicit def queryToJson( + query: Query + ): JsonNode = { + JacksonBuilder.toNode( + SearchBodyBuilderFn( + ElasticApi.search("") query { + query + } + ).value + ) match { + case Left(node: JsonNode) => + if (node.has("query")) { + node.get("query") + } else { + node + } + case Right(_) => NullNode.instance + } + } + + implicit def criteriaToQuery(criteria: Criteria)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Query = { + ElasticCriteria(criteria).asQuery() + } + + implicit def criteriaToNode(criteria: Criteria)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): JsonNode = { + queryToJson(criteriaToQuery(criteria)) + } + implicit def filterToQuery( filter: ElasticFilter ): ElasticBridge = { @@ -1015,7 +1087,10 @@ package object bridge { implicit def sqlQueryToAggregations( query: SelectStatement - )(implicit timestamp: Long): Seq[ElasticAggregation] = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Seq[ElasticAggregation] = { import query._ statement .map { diff --git a/build.sbt b/build.sbt index 7fe63779..c6ef7f6e 100644 --- a/build.sbt +++ b/build.sbt @@ -1,5 +1,6 @@ +import app.softnetwork.Publish +import scala.collection.Seq import SoftClient4es.* -import app.softnetwork.* import sbt.Def import sbtbuildinfo.BuildInfoKeys.buildInfoObject @@ -19,7 +20,7 @@ ThisBuild / organization := "app.softnetwork" name := "softclient4es" -ThisBuild / version := "0.15.0" +ThisBuild / version := "0.16-SNAPSHOT" ThisBuild / scalaVersion := scala213 @@ -99,6 +100,14 @@ ThisBuild / libraryDependencySchemes += "org.scala-lang.modules" %% "scala-xml" Test / parallelExecution := false +lazy val licensing = project + .in(file("licensing")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + moduleSettings + ) + lazy val sql = project .in(file("sql")) .configs(IntegrationTest) @@ -142,6 +151,9 @@ lazy val core = project .dependsOn( macros % "compile->compile;test->test;it->it" ) + .dependsOn( + licensing % "compile->compile;test->test;it->it" + ) lazy val persistence = project .in(file("persistence")) @@ -461,6 +473,7 @@ lazy val root = project crossScalaVersions := Nil ) .aggregate( + licensing, sql, bridge, macros, diff --git a/core/src/main/resources/META-INF/services/app.softnetwork.elastic.client.ExtensionSpi b/core/src/main/resources/META-INF/services/app.softnetwork.elastic.client.ExtensionSpi new file mode 100644 index 00000000..d2e812cf --- /dev/null +++ b/core/src/main/resources/META-INF/services/app.softnetwork.elastic.client.ExtensionSpi @@ -0,0 +1,2 @@ +app.softnetwork.elastic.client.extensions.CoreDdlExtension +app.softnetwork.elastic.client.extensions.CoreDqlExtension \ No newline at end of file diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala index 964a4967..c4542683 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala @@ -24,7 +24,6 @@ import org.json4s.jackson import org.json4s.jackson.Serialization import org.slf4j.Logger -import java.io.Closeable import scala.language.{implicitConversions, postfixOps, reflectiveCalls} /** Created by smanciot on 28/06/2018. @@ -49,6 +48,9 @@ trait ElasticClientApi with SerializationApi with PipelineApi with TemplateApi + with EnrichPolicyApi + with TransformApi + with ExtensionApi with GatewayApi with ClientCompanion { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index a45c174a..2327ccd4 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -22,15 +22,15 @@ import akka.stream.scaladsl.{Flow, Source} import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.schema.Index -import app.softnetwork.elastic.sql.{query, schema} +import app.softnetwork.elastic.schema.{Index, IndexMappings} +import app.softnetwork.elastic.sql.{query, schema, PainlessContextType} import app.softnetwork.elastic.sql.query.{ DqlStatement, SQLAggregation, SelectStatement, SingleSearch } -import app.softnetwork.elastic.sql.schema.{Schema, TableAlias} +import app.softnetwork.elastic.sql.schema.{Schema, TableAlias, TransformCreationStatus} import com.typesafe.config.Config import org.json4s.Formats import org.slf4j.{Logger, LoggerFactory} @@ -610,6 +610,9 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { ): ElasticResult[Boolean] = delegate.updateMapping(index, mapping, settings) + override def allMappings: ElasticResult[Map[String, IndexMappings]] = + delegate.allMappings + /** Migrate an existing index to a new mapping. * * Process: @@ -645,6 +648,10 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { delegate.executeGetMapping(index) } + override private[client] def executeGetAllMappings(): ElasticResult[Map[String, String]] = { + delegate.executeGetAllMappings() + } + // ==================== RefreshApi ==================== /** Refresh the index to make sure all documents are indexed and searchable. @@ -1322,10 +1329,13 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { )(implicit formats: Formats): ElasticResult[Seq[(U, Seq[I])]] = delegate.multisearchWithInnerHits[U, I](elasticQueries, innerField) - override private[client] implicit def sqlSearchRequestToJsonQuery( + override private[client] implicit def singleSearchToJsonQuery( sqlSearch: SingleSearch - )(implicit timestamp: Long): String = - delegate.sqlSearchRequestToJsonQuery(sqlSearch) + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): String = + delegate.singleSearchToJsonQuery(sqlSearch) override private[client] def executeSingleSearch( elasticQuery: ElasticQuery @@ -1795,4 +1805,82 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { system: ActorSystem ): Future[ElasticResult[QueryResult]] = delegate.run(statement) + + // ==================== Transform (delegate) ==================== + + override def createTransform( + config: schema.TransformConfig, + start: Boolean + ): ElasticResult[TransformCreationStatus] = + delegate.createTransform(config, start) + + override def deleteTransform(transformId: String, force: Boolean): ElasticResult[Boolean] = + delegate.deleteTransform(transformId, force) + + override def startTransform(transformId: String): ElasticResult[Boolean] = + delegate.startTransform(transformId) + + override def stopTransform( + transformId: String, + force: Boolean, + waitForCompletion: Boolean + ): ElasticResult[Boolean] = + delegate.stopTransform(transformId, force, waitForCompletion) + + override def getTransformStats( + transformId: String + ): ElasticResult[Option[schema.TransformStats]] = + delegate.getTransformStats(transformId) + + override private[client] def executeCreateTransform( + config: schema.TransformConfig, + start: Boolean + ): ElasticResult[Boolean] = + delegate.executeCreateTransform(config, start) + + override private[client] def executeDeleteTransform( + transformId: String, + force: Boolean + ): ElasticResult[Boolean] = + delegate.executeDeleteTransform(transformId, force) + + override private[client] def executeStartTransform(transformId: String): ElasticResult[Boolean] = + delegate.executeStartTransform(transformId) + + override private[client] def executeStopTransform( + transformId: String, + force: Boolean, + waitForCompletion: Boolean + ): ElasticResult[Boolean] = + delegate.executeStopTransform(transformId, force, waitForCompletion) + + override private[client] def executeGetTransformStats( + transformId: String + ): ElasticResult[Option[schema.TransformStats]] = + delegate.executeGetTransformStats(transformId) + // ==================== Enrich policy (delegate) ==================== + + override def createEnrichPolicy(policy: schema.EnrichPolicy): ElasticResult[Boolean] = + delegate.createEnrichPolicy(policy) + + override def deleteEnrichPolicy(policyName: String): ElasticResult[Boolean] = + delegate.deleteEnrichPolicy(policyName) + + override def executeEnrichPolicy(policyName: String): ElasticResult[String] = + delegate.executeEnrichPolicy(policyName) + + override private[client] def executeCreateEnrichPolicy( + policy: schema.EnrichPolicy + ): ElasticResult[Boolean] = + delegate.executeCreateEnrichPolicy(policy) + + override private[client] def executeDeleteEnrichPolicy( + policyName: String + ): ElasticResult[Boolean] = + delegate.executeDeleteEnrichPolicy(policyName) + + override private[client] def executeExecuteEnrichPolicy( + policyName: String + ): ElasticResult[String] = + delegate.executeExecuteEnrichPolicy(policyName) } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala index 24e9ad72..7d239c29 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala @@ -149,4 +149,22 @@ object ElasticsearchVersion { def supportsDeletionByQueryOnClosedIndices(version: String): Boolean = { isAtLeast(version, 7, 5) } + + /** Check if enrich processor is supported (ES >= 7.5) + */ + def supportsEnrich(version: String): Boolean = { + isAtLeast(version, 7, 5) + } + + /** Check if latest transform features are supported (ES >= 7.8) + */ + def supportsLatestTransform(version: String): Boolean = { + isAtLeast(version, 7, 8) + } + + /** Check if materialized views are supported (enrich + latest transform) + */ + def supportsMaterializedView(version: String): Boolean = { + supportsEnrich(version) && supportsLatestTransform(version) + } } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/EnrichPolicyApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/EnrichPolicyApi.scala new file mode 100644 index 00000000..fedda016 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/EnrichPolicyApi.scala @@ -0,0 +1,90 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.result.{ + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess +} +import app.softnetwork.elastic.sql.schema.EnrichPolicy + +trait EnrichPolicyApi extends ElasticClientHelpers { _: VersionApi => + + private def checkVersionForEnrichPolicy(): ElasticResult[Boolean] = { + val elasticVersion = version match { + case ElasticSuccess(value) => value + case ElasticFailure(error) => + val failure = + s"Cannot retrieve elastic version to check enrich policy support: ${error.getMessage}" + logger.error(s"❌ $failure") + return ElasticFailure( + ElasticError( + message = failure, + operation = Some("CheckVersionForEnrichPolicy"), + statusCode = Some(400) + ) + ) + } + if (!ElasticsearchVersion.supportsEnrich(elasticVersion)) { + val failure = s"Enrich policies are not supported in elastic version $elasticVersion" + logger.error(s"❌ $failure") + return ElasticFailure( + ElasticError( + message = failure, + operation = Some("CheckVersionForEnrichPolicy"), + statusCode = Some(400) + ) + ) + } + ElasticSuccess(true) + } + + def createEnrichPolicy(policy: EnrichPolicy): ElasticResult[Boolean] = { + checkVersionForEnrichPolicy() match { + case ElasticSuccess(_) => // continue + case failure @ ElasticFailure(_) => return failure + } + logger.info(s"🔧 Creating enrich policy: ${policy.name}") + executeCreateEnrichPolicy(policy) + } + + def deleteEnrichPolicy(policyName: String): ElasticResult[Boolean] = { + checkVersionForEnrichPolicy() match { + case ElasticSuccess(_) => // continue + case failure @ ElasticFailure(_) => return failure + } + logger.info(s"🔧 Deleting enrich policy: $policyName") + executeDeleteEnrichPolicy(policyName) + } + + def executeEnrichPolicy(policyName: String): ElasticResult[String] = { + checkVersionForEnrichPolicy() match { + case ElasticSuccess(_) => // continue + case failure @ ElasticFailure(_) => return failure + } + logger.info(s"🔧 Executing enrich policy: $policyName") + executeExecuteEnrichPolicy(policyName) + } + + private[client] def executeCreateEnrichPolicy(policy: EnrichPolicy): ElasticResult[Boolean] + + private[client] def executeDeleteEnrichPolicy(policyName: String): ElasticResult[Boolean] + + private[client] def executeExecuteEnrichPolicy(policyName: String): ElasticResult[String] +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ExtensionApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ExtensionApi.scala new file mode 100644 index 00000000..56b857e9 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/ExtensionApi.scala @@ -0,0 +1,28 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.licensing.{DefaultLicenseManager, LicenseManager} + +trait ExtensionApi { self: ElasticClientApi => + + // ✅ Inject license manager (overridable) + def licenseManager: LicenseManager = new DefaultLicenseManager() + + /** Extension registry (lazy loaded) */ + lazy val extensionRegistry: ExtensionRegistry = new ExtensionRegistry(config, licenseManager) +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ExtensionRegistry.scala b/core/src/main/scala/app/softnetwork/elastic/client/ExtensionRegistry.scala new file mode 100644 index 00000000..d10c17ea --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/ExtensionRegistry.scala @@ -0,0 +1,80 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.licensing.LicenseManager +import app.softnetwork.elastic.sql.query.Statement +import com.typesafe.config.Config + +/** Registry for managing loaded extensions. + */ +class ExtensionRegistry( + config: Config, + licenseManager: LicenseManager +) { + + import scala.jdk.CollectionConverters._ + import java.util.ServiceLoader + + private val logger = org.slf4j.LoggerFactory.getLogger(getClass) + + /** All loaded extensions. + */ + lazy val extensions: Seq[ExtensionSpi] = { + val loader = ServiceLoader.load(classOf[ExtensionSpi]) + + val loaded = loader.iterator().asScala.toSeq.flatMap { ext => + logger.info( + s"🔌 Discovered extension: ${ext.extensionName} v${ext.version} (priority: ${ext.priority})" + ) + + ext.initialize(config, licenseManager) match { + case Right(_) => + logger.info(s"✅ Extension ${ext.extensionName} initialized successfully") + Some(ext) + + case Left(error) => + logger.warn(s"⚠️ Failed to initialize extension ${ext.extensionName}: $error") + None + } + } + + // ✅ Sort by priority (lower = higher priority) + loaded.sortBy(_.priority) + } + + /** Find extension that can handle a statement. + */ + def findHandler(statement: Statement): Option[ExtensionSpi] = { + extensions.find(_.canHandle(statement)) + } + + /** Get all supported syntax. + */ + def allSupportedSyntax: Seq[String] = { + extensions.flatMap(_.supportedSyntax).distinct + } + + /** Shutdown all extensions. + */ + def shutdown(): Unit = { + extensions.foreach { ext => + logger.info(s"🔌 Shutting down extension: ${ext.extensionName}") + ext.shutdown() + } + } +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ExtensionSpi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ExtensionSpi.scala new file mode 100644 index 00000000..52576194 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/ExtensionSpi.scala @@ -0,0 +1,93 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.actor.ActorSystem +import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.licensing.LicenseManager +import app.softnetwork.elastic.sql.query.Statement +import com.typesafe.config.Config + +import scala.concurrent.Future + +/** Service Provider Interface for Elasticsearch extensions. + * + * Extensions can add new capabilities to the client (e.g., materialized views, advanced analytics, + * custom drivers) while keeping the core API stable. + */ +trait ExtensionSpi { + + /** Unique identifier for this extension. + */ + def extensionId: String + + /** Human-readable name. + */ + def extensionName: String + + /** Version of the extension. + */ + def version: String + + /** Priority for extension resolution (lower = higher priority). + * + * Default: 100 (core extensions) Premium extensions should use lower values (e.g., 50, 10) + */ + def priority: Int = 100 + + /** Initialize the extension with configuration and license manager. + * + * @param config + * Extension-specific configuration + * @param licenseManager + * License manager for checking quotas + * @return + * Success or error message + */ + def initialize( + config: Config, + licenseManager: LicenseManager + ): Either[String, Unit] + + /** Check if this extension can handle a given SQL statement. + */ + def canHandle(statement: Statement): Boolean + + /** Execute a statement handled by this extension. + * + * @param statement + * SQL statement to execute + * @param client + * Elasticsearch client API for low-level operations + * @return + * Query result + */ + def execute( + statement: Statement, + client: ElasticClientApi + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] + + /** List of SQL keywords/syntax added by this extension. + * + * Example: ["CREATE MATERIALIZED VIEW", "REFRESH MATERIALIZED VIEW"] + */ + def supportedSyntax: Seq[String] + + /** Shutdown hook for cleanup. + */ + def shutdown(): Unit = () +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala index 69b8f80c..c5184070 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala @@ -53,6 +53,7 @@ import app.softnetwork.elastic.sql.query.{ ShowCreateTable, ShowPipeline, ShowTable, + ShowTables, SingleSearch, Statement, TableStatement, @@ -277,7 +278,7 @@ class PipelineExecutor(api: PipelineApi, logger: Logger) extends DdlExecutor[Pip case ElasticSuccess(pipeline) => logger.info(s"✅ Retrieved pipeline ${describe.name}.") Future.successful( - ElasticResult.success(QueryRows(pipeline.processors.map(_.properties))) + ElasticResult.success(QueryRows(pipeline.describe)) ) case ElasticFailure(elasticError) => Future.successful( @@ -317,6 +318,26 @@ class TableExecutor( // handle TABLE statement statement match { // handle SHOW TABLE statement + case ShowTables => + api.allMappings match { + case ElasticSuccess(mappings) => + logger.info("✅ Retrieved all tables.") + Future.successful( + ElasticResult.success( + QueryRows( + mappings.map { case (index, mappings) => + Map("name" -> index, "type" -> mappings.tableType.name.toUpperCase) + }.toSeq + ) + ) + ) + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("tables")) + ) + ) + } case show: ShowTable => api.loadSchema(show.table) match { case ElasticSuccess(schema) => @@ -348,7 +369,7 @@ class TableExecutor( api.loadSchema(describe.table) match { case ElasticSuccess(schema) => logger.info(s"✅ Retrieved schema for index ${describe.table}.") - Future.successful(ElasticResult.success(QueryRows(schema.columns.flatMap(_.asMap)))) + Future.successful(ElasticResult.success(QueryRows(schema.describe))) case ElasticFailure(elasticError) => Future.successful( ElasticFailure( @@ -408,7 +429,6 @@ class TableExecutor( // 4) Index not exists → creation case ElasticSuccess(false) => // proceed with creation - logger.info(s"✅ Creating index $indexName.") createNonExistentIndex(indexName, create, partitioned, single) // 5) Error on indexExists @@ -915,7 +935,6 @@ class TableExecutor( ) match { case success @ ElasticSuccess(true) => // index created successfully - logger.info(s"✅ Index $indexName created successfully.") success case ElasticSuccess(_) => // index creation failed @@ -1155,15 +1174,7 @@ class DdlRouterExecutor( } trait GatewayApi extends ElasticClientHelpers { - _: IndicesApi - with PipelineApi - with MappingApi - with SettingsApi - with AliasApi - with TemplateApi - with SearchApi - with ScrollApi - with VersionApi => + self: ElasticClientApi => lazy val dqlExecutor = new DqlExecutor( api = this, @@ -1255,29 +1266,38 @@ trait GatewayApi extends ElasticClientHelpers { def run( statement: Statement )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { - statement match { + implicit val ec: ExecutionContext = system.dispatcher - case dql: DqlStatement => - dqlExecutor.execute(dql) + // ✅ TRY EXTENSIONS FIRST + extensionRegistry.findHandler(statement) match { + case Some(extension) => + logger.info(s"🔌 Routing to extension: ${extension.extensionName}") + extension.execute(statement, self) // ✅ Pass full client API - // handle DML statements - case dml: DmlStatement => - dmlExecutor.execute(dml) + case None => + // ✅ FALLBACK TO STANDARD EXECUTORS + statement match { + case dql: DqlStatement => + logger.debug("🔧 Executing DQL with base executor") + dqlExecutor.execute(dql) - // handle DDL statements - case ddl: DdlStatement => - ddlExecutor.execute(ddl) + case dml: DmlStatement => + logger.debug("🔧 Executing DML with base executor") + dmlExecutor.execute(dml) - case _ => - // unsupported SQL statement - val error = - ElasticError( - message = s"Unsupported SQL statement: $statement", - statusCode = Some(400), - operation = Some("schema") - ) - logger.error(s"❌ ${error.message}") - Future.successful(ElasticFailure(error)) + case ddl: DdlStatement => + logger.debug("🔧 Executing DDL with base executor") + ddlExecutor.execute(ddl) + + case _ => + val error = ElasticError( + message = s"Unsupported SQL statement: $statement", + statusCode = Some(400), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + } } } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala index 680f4ee9..a895e6e8 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala @@ -21,6 +21,7 @@ import akka.stream.scaladsl.Source import app.softnetwork.elastic.client.bulk.BulkOptions import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.PainlessContextType import app.softnetwork.elastic.sql.parser.Parser import app.softnetwork.elastic.sql.query.{ Delete, @@ -1671,8 +1672,9 @@ trait IndicesApi extends ElasticClientHelpers { * @return * JSON string representation of the query */ - private[client] implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit - timestamp: Long + private[client] implicit def singleSearchToJsonQuery(sqlSearch: SingleSearch)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query ): String // ======================================================================== diff --git a/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala index 088d0bb0..5c878125 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala @@ -22,6 +22,7 @@ import app.softnetwork.elastic.client.result.{ ElasticResult, ElasticSuccess } +import app.softnetwork.elastic.schema.IndexMappings import app.softnetwork.elastic.sql.schema.TableAlias import com.fasterxml.jackson.databind.node.ObjectNode import com.google.gson.JsonParser @@ -154,6 +155,20 @@ trait MappingApi extends ElasticClientHelpers { executeGetMapping(index) } + /** Get the mappings of all indices. + * @return + * a map of index names to their mappings + */ + def allMappings: ElasticResult[Map[String, IndexMappings]] = { + // Get mappings for all indices + executeGetAllMappings().map { mappingsMap => + mappingsMap.map { case (index, mappingJson) => + val indexMappings = IndexMappings(mappingJson) + (index, indexMappings) + } + } + } + /** Get the mapping properties of an index. * @param index * - the name of the index to get the mapping properties for @@ -392,4 +407,6 @@ trait MappingApi extends ElasticClientHelpers { private[client] def executeSetMapping(index: String, mapping: String): ElasticResult[Boolean] private[client] def executeGetMapping(index: String): ElasticResult[String] + + private[client] def executeGetAllMappings(): ElasticResult[Map[String, String]] } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala index d74cf353..f1f0afa0 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala @@ -26,7 +26,7 @@ import app.softnetwork.elastic.client.bulk.{ FailedDocument, SuccessfulDocument } -import app.softnetwork.elastic.sql.query +import app.softnetwork.elastic.sql.{query, schema, PainlessContextType} import app.softnetwork.elastic.sql.query.SQLAggregation import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ @@ -214,9 +214,12 @@ trait NopeClientApi extends ElasticClientApi { * @return * JSON string representation of the query */ - override private[client] implicit def sqlSearchRequestToJsonQuery( + override private[client] implicit def singleSearchToJsonQuery( sqlSearch: query.SingleSearch - )(implicit timestamp: Long): String = "{\"query\": {\"match_all\": {}}}" + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): String = "{\"query\": {\"match_all\": {}}}" override private[client] def executeUpdateSettings( index: String, @@ -376,4 +379,49 @@ trait NopeClientApi extends ElasticClientApi { /** Conversion BulkActionType -> BulkItem */ override private[client] def actionToBulkItem(action: BulkActionType): BulkItem = throw new UnsupportedOperationException + + override private[client] def executeGetAllMappings(): ElasticResult[Map[String, String]] = + ElasticResult.success(Map.empty) + + override private[client] def executeCreateEnrichPolicy( + policy: schema.EnrichPolicy + ): ElasticResult[Boolean] = + ElasticSuccess(false) + + override private[client] def executeDeleteEnrichPolicy( + policyName: String + ): ElasticResult[Boolean] = + ElasticSuccess(false) + + override private[client] def executeExecuteEnrichPolicy( + policyName: String + ): ElasticResult[String] = + ElasticSuccess("") + + override private[client] def executeCreateTransform( + config: schema.TransformConfig, + start: Boolean + ): ElasticResult[Boolean] = + ElasticSuccess(false) + + override private[client] def executeDeleteTransform( + transformId: String, + force: Boolean + ): ElasticResult[Boolean] = + ElasticSuccess(false) + + override private[client] def executeStartTransform(transformId: String): ElasticResult[Boolean] = + ElasticSuccess(false) + + override private[client] def executeStopTransform( + transformId: String, + force: Boolean, + waitForCompletion: Boolean + ): ElasticResult[Boolean] = + ElasticSuccess(false) + + override private[client] def executeGetTransformStats( + transformId: String + ): ElasticResult[Option[schema.TransformStats]] = + ElasticSuccess(None) } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala index 7977c791..19e2df5c 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala @@ -22,6 +22,7 @@ import app.softnetwork.elastic.client.result.{ ElasticResult, ElasticSuccess } +import app.softnetwork.elastic.sql.PainlessContextType import app.softnetwork.elastic.sql.macros.SQLQueryMacros import app.softnetwork.elastic.sql.query.{ DqlStatement, @@ -1035,8 +1036,9 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * @return * JSON string representation of the query */ - private[client] implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit - timestamp: Long + private[client] implicit def singleSearchToJsonQuery(sqlSearch: SingleSearch)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query ): String private def parseInnerHits[M: Manifest: ClassTag, I: Manifest: ClassTag]( diff --git a/core/src/main/scala/app/softnetwork/elastic/client/TransformApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/TransformApi.scala new file mode 100644 index 00000000..e15713d2 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/TransformApi.scala @@ -0,0 +1,178 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.result.{ + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess +} +import app.softnetwork.elastic.sql.schema.{TransformConfig, TransformCreationStatus, TransformStats} + +trait TransformApi extends ElasticClientHelpers { + + /** Create a transform with the given configuration. If start is true, the transform will be + * started immediately after creation. + * @param config + * The configuration of the transform to create. + * @param start + * Whether to start the transform immediately after creation. + * @return + * ElasticResult[TransformCreationStatus] indicating whether the create operation and its + * optional start were successful. + */ + def createTransform( + config: TransformConfig, + start: Boolean = false + ): ElasticResult[TransformCreationStatus] = { + logger.info(s"Creating transform [${config.id}]...") + executeCreateTransform(config, start) match { + case ElasticSuccess(true) if start => + logger.info(s"✅ Transform [${config.id}] created successfully. Starting transform...") + startTransform(config.id) match { + case ElasticSuccess(status) => + ElasticSuccess(TransformCreationStatus(created = true, started = Some(status))) + case failure @ ElasticFailure(_) => + failure + } + case ElasticSuccess(status) => + ElasticSuccess(TransformCreationStatus(created = status)) + case failure @ ElasticFailure(_) => failure + } + } + + /** Delete the transform with the given ID. If force is true, the transform will be deleted even + * if it is currently running. + * @param transformId + * The ID of the transform to delete. + * @param force + * Whether to force delete the transform. + * @return + * ElasticResult[Boolean] indicating whether the delete operation was successful. + */ + def deleteTransform(transformId: String, force: Boolean = false): ElasticResult[Boolean] = { + logger.info(s"Deleting transform [$transformId]...") + executeDeleteTransform(transformId, force) match { + case success @ ElasticSuccess(true) => + logger.info(s"✅ Transform [$transformId] deleted successfully.") + success + case success @ ElasticSuccess(false) => + logger.warn(s"⚠️ Transform [$transformId] could not be deleted.") + success + case failure @ ElasticFailure(error) => + logger.error(s"❌ Failed to delete transform [$transformId]: ${error.message}") + failure + } + } + + /** Start the transform with the given ID. + * @param transformId + * The ID of the transform to start. + * @return + * ElasticResult[Boolean] indicating whether the start operation was successful. + */ + def startTransform(transformId: String): ElasticResult[Boolean] = { + logger.info(s"Starting transform [$transformId]...") + executeStartTransform(transformId) match { + case success @ ElasticSuccess(true) => + logger.info(s"✅ Transform [$transformId] started successfully.") + success + case success @ ElasticSuccess(false) => + logger.warn(s"⚠️ Transform [$transformId] could not be started.") + success + case failure @ ElasticFailure(error) => + logger.error(s"❌ Failed to start transform [$transformId]: ${error.message}") + failure + } + } + + /** Stop the transform with the given ID. If force is true, the transform will be stopped + * immediately. If waitForCompletion is true, the method will wait until the transform is fully + * stopped before returning. + * @param transformId + * The ID of the transform to stop. + * @param force + * Whether to force stop the transform. + * @param waitForCompletion + * Whether to wait for the transform to be fully stopped. + * @return + * ElasticResult[Boolean] indicating whether the stop operation was successful. + */ + def stopTransform( + transformId: String, + force: Boolean = false, + waitForCompletion: Boolean = true + ): ElasticResult[Boolean] = { + logger.info(s"Stopping transform [$transformId]...") + executeStopTransform(transformId, force, waitForCompletion) match { + case success @ ElasticSuccess(true) => + logger.info(s"✅ Transform [$transformId] stopped successfully.") + success + case success @ ElasticSuccess(false) => + logger.warn(s"⚠️ Transform [$transformId] could not be stopped.") + success + case failure @ ElasticFailure(error) => + logger.error(s"❌ Failed to stop transform [$transformId]: ${error.message}") + failure + } + } + + /** Get the statistics of the transform with the given ID. + * @param transformId + * The ID of the transform to get statistics for. + * @return + * ElasticResult[Option[TransformStats]\] containing the transform statistics if available. + */ + def getTransformStats(transformId: String): ElasticResult[Option[TransformStats]] = { + logger.info(s"Getting stats for transform [$transformId]...") + executeGetTransformStats(transformId) match { + case success @ ElasticSuccess(Some(_)) => + logger.info(s"✅ Retrieved stats for transform [$transformId] successfully.") + success + case success @ ElasticSuccess(None) => + logger.warn(s"⚠️ No stats found for transform [$transformId].") + success + case failure @ ElasticFailure(error) => + logger.error(s"❌ Failed to get stats for transform [$transformId]: ${error.message}") + failure + } + } + + private[client] def executeCreateTransform( + config: TransformConfig, + start: Boolean + ): ElasticResult[Boolean] + + private[client] def executeDeleteTransform( + transformId: String, + force: Boolean + ): ElasticResult[Boolean] + + private[client] def executeStartTransform(transformId: String): ElasticResult[Boolean] + + private[client] def executeStopTransform( + transformId: String, + force: Boolean, + waitForCompletion: Boolean + ): ElasticResult[Boolean] + + private[client] def executeGetTransformStats( + transformId: String + ): ElasticResult[Option[TransformStats]] + +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDdlExtension.scala b/core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDdlExtension.scala new file mode 100644 index 00000000..33a59092 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDdlExtension.scala @@ -0,0 +1,106 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.extensions + +import akka.actor.ActorSystem +import app.softnetwork.elastic.client._ +import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.licensing._ +import app.softnetwork.elastic.sql.query._ +import com.typesafe.config.Config + +import scala.concurrent.{ExecutionContext, Future} + +//format:off +/** Core extension for basic DDL operations. + * + * {{{ + * ✅ OSS (always loaded) + * ✅ Handles simple DDL (CREATE TABLE, DROP TABLE, etc.) + * ✅ Does NOT handle materialized views (premium extension) + * }}} + */ +//format:on +class CoreDdlExtension extends ExtensionSpi { + + private var licenseManager: Option[LicenseManager] = None + private val logger = org.slf4j.LoggerFactory.getLogger(getClass) + + override def extensionId: String = "core-ddl" + override def extensionName: String = "Core DDL" + override def version: String = "0.1.0" + + override def initialize( + config: Config, + manager: LicenseManager + ): Either[String, Unit] = { + logger.info("🔌 Initializing Core DDL extension") + licenseManager = Some(manager) + Right(()) + } + + override def priority: Int = 100 + + override def canHandle(statement: Statement): Boolean = statement match { + case ddl: DdlStatement => + // ✅ Handle basic DDL, but NOT materialized views + ddl match { + case _: MaterializedViewStatement => false + case _ => true + } + case _ => false + } + + override def execute( + statement: Statement, + client: ElasticClientApi + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + + statement match { + case ddl: DdlStatement => + // ✅ No quota checks for basic DDL + client.ddlExecutor.execute(ddl) + + case _ => + Future.successful( + ElasticFailure( + ElasticError( + message = "Statement not supported by this extension", + statusCode = Some(400), + operation = Some("extension") + ) + ) + ) + } + } + + override def supportedSyntax: Seq[String] = Seq( + "CREATE TABLE", + "DROP TABLE", + "ALTER TABLE", + "TRUNCATE TABLE", + "SHOW CREATE TABLE", + "SHOW TABLE", + "DESCRIBE TABLE", + "CREATE PIPELINE", + "ALTER PIPELINE", + "DROP PIPELINE", + "SHOW PIPELINE", + "DESCRIBE PIPELINE" + ) +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtension.scala b/core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtension.scala new file mode 100644 index 00000000..ff4b0b4a --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtension.scala @@ -0,0 +1,139 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.extensions + +import akka.actor.ActorSystem +import app.softnetwork.elastic.client._ +import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.licensing._ +import app.softnetwork.elastic.sql.query._ +import com.typesafe.config.Config + +import scala.concurrent.{ExecutionContext, Future} + +//format:off +/** Core extension for DQL quota enforcement. + * + * {{{ + * ✅ OSS (always loaded) + * ✅ Applies Community quotas by default + * ✅ Can be upgraded with Pro license + * }}} + */ +//format:on +class CoreDqlExtension extends ExtensionSpi { + + private var licenseManager: Option[LicenseManager] = None + private val logger = org.slf4j.LoggerFactory.getLogger(getClass) + + override def extensionId: String = "core-dql" + override def extensionName: String = "Core DQL Quotas" + override def version: String = "0.1.0" + + override def initialize( + config: Config, + manager: LicenseManager + ): Either[String, Unit] = { + logger.info("🔌 Initializing Core DQL extension") + licenseManager = Some(manager) + Right(()) + } + + // ✅ Priority: 100 (lower = higher priority) + // Built-in extensions should have lower priority than premium ones + override def priority: Int = 100 + + override def canHandle(statement: Statement): Boolean = statement match { + case _: DqlStatement => true + case _ => false + } + + override def execute( + statement: Statement, + client: ElasticClientApi + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + + statement match { + case dql: DqlStatement => + licenseManager match { + case Some(manager) => + checkQuotasAndExecute(dql, manager, client) + case None => + // No license manager, execute without checks + client.dqlExecutor.execute(dql) + } + + case _ => + Future.successful( + ElasticFailure( + ElasticError( + message = "Statement not supported by this extension", + statusCode = Some(400), + operation = Some("extension") + ) + ) + ) + } + } + + override def supportedSyntax: Seq[String] = Seq( + "SELECT ... FROM ... WHERE ... GROUP BY ... HAVING ... LIMIT ..." + ) + + // ════════════════════════════════════════════════════════════════════ + // ✅ QUOTA CHECK LOGIC (in extension, not in core) + // ════════════════════════════════════════════════════════════════════ + + private def checkQuotasAndExecute( + dql: DqlStatement, + manager: LicenseManager, + client: ElasticClientApi + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + + val quota = manager.quotas + + dql match { + case single: SingleSearch => + single.limit match { + case Some(l) if quota.maxQueryResults.exists(_ < l.limit) => + logger.warn( + s"⚠️ Query result limit (${l.limit}) exceeds license quota (${quota.maxQueryResults.get})" + ) + Future.successful( + ElasticFailure( + ElasticError( + message = + s"Query result limit (${l.limit}) exceeds license quota (${quota.maxQueryResults.get}). Upgrade to Pro license.", + statusCode = Some(402), + operation = Some("license") + ) + ) + ) + + case _ => + // Quota OK, execute + client.dqlExecutor.execute(single) + } + + case _ => + // Other DQL types (no quota check needed) + client.dqlExecutor.execute(dql) + } + } +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala index 7b78425f..edf0af60 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala @@ -30,10 +30,10 @@ import app.softnetwork.elastic.client.{ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.schema.{Index, IndexMappings} import app.softnetwork.elastic.sql.{query, schema} import app.softnetwork.elastic.sql.query.{DqlStatement, SQLAggregation, SelectStatement} -import app.softnetwork.elastic.sql.schema.{Schema, TableAlias} +import app.softnetwork.elastic.sql.schema.{Schema, TableAlias, TransformCreationStatus} import org.json4s.Formats import scala.concurrent.{ExecutionContext, Future} @@ -454,6 +454,11 @@ class MetricsElasticClient( delegate.updateMapping(index, mapping, settings) } + override def allMappings: ElasticResult[Map[String, IndexMappings]] = + measureResult("allMappings") { + delegate.allMappings + } + // ==================== RefreshApi ==================== override def refresh(index: String): ElasticResult[Boolean] = { @@ -1365,4 +1370,58 @@ class MetricsElasticClient( } } + // ==================== Transform (delegate) ==================== + + override def createTransform( + config: schema.TransformConfig, + start: Boolean + ): ElasticResult[TransformCreationStatus] = + measureResult("createTransform") { + delegate.createTransform(config, start) + } + + override def deleteTransform(transformId: String, force: Boolean): ElasticResult[Boolean] = + measureResult("deleteTransform") { + delegate.deleteTransform(transformId, force) + } + + override def startTransform(transformId: String): ElasticResult[Boolean] = { + measureResult("startTransform") { + delegate.startTransform(transformId) + } + } + + override def stopTransform( + transformId: String, + force: Boolean, + waitForCompletion: Boolean + ): ElasticResult[Boolean] = + measureResult("stopTransform") { + delegate.stopTransform(transformId, force, waitForCompletion) + } + + override def getTransformStats( + transformId: String + ): ElasticResult[Option[schema.TransformStats]] = + measureResult("getTransformStats") { + delegate.getTransformStats(transformId) + } + + // ==================== Enrich policy (delegate) ==================== + + override def createEnrichPolicy(policy: schema.EnrichPolicy): ElasticResult[Boolean] = + measureResult("createEnrichPolicy") { + delegate.createEnrichPolicy(policy) + } + + override def deleteEnrichPolicy(policyName: String): ElasticResult[Boolean] = + measureResult("deleteEnrichPolicy") { + delegate.deleteEnrichPolicy(policyName) + } + + override def executeEnrichPolicy(policyName: String): ElasticResult[String] = + measureResult("executeEnrichPolicy") { + delegate.executeEnrichPolicy(policyName) + } + } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala index 3582acfe..91362e78 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala @@ -401,7 +401,9 @@ package object result { stream: Source[(Map[String, Any], ScrollMetrics), NotUsed] ) extends QueryResult - case class QueryStructured(response: ElasticResponse) extends QueryResult + case class QueryStructured(response: ElasticResponse) extends QueryResult { + def asQueryRows: QueryRows = QueryRows(response.results) + } // -------------------- // DML (INSERT / UPDATE / DELETE) diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index 792d3e0d..e6cacfa2 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.PainlessContext +import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType} import app.softnetwork.elastic.sql.`type`.SQLTemporal import app.softnetwork.elastic.sql.query.{ Asc, @@ -100,7 +100,10 @@ object ElasticAggregation { having: Option[Criteria], bucketsDirection: Map[String, SortOrder], allAggregations: Map[String, SQLAggregation] - )(implicit timestamp: Long): ElasticAggregation = { + )(implicit + timestamp: Long, + contextType: PainlessContextType + ): ElasticAggregation = { import sqlAgg._ val sourceField = identifier.path @@ -153,14 +156,14 @@ object ElasticAggregation { buildScript: (String, Script) => Aggregation ): Aggregation = { if (transformFuncs.nonEmpty) { - val context = PainlessContext() + val context = PainlessContext(context = contextType) val scriptSrc = identifier.painless(Some(context)) val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) } else { aggType match { case th: WindowFunction if th.shouldBeScripted => - val context = PainlessContext() + val context = PainlessContext(context = contextType) val scriptSrc = th.identifier.painless(Some(context)) val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) @@ -348,7 +351,10 @@ object ElasticAggregation { having: Option[Criteria], nested: Option[NestedElement], allElasticAggregations: Seq[ElasticAggregation] - )(implicit timestamp: Long): Seq[Aggregation] = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Seq[Aggregation] = { for (tree <- buckets) yield { val treeNodes = tree.sortBy(_.level).reverse.foldLeft(Seq.empty[NodeAggregation]) { (current, node) => @@ -364,7 +370,7 @@ object ElasticAggregation { val aggScript = if (!bucket.isBucketScript && bucket.shouldBeScripted) { - val context = PainlessContext() + val context = PainlessContext(context = contextType) val painless = bucket.painless(Some(context)) Some(now(Script(s"$context$painless").lang("painless"))) } else { diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala index ad5ad8ad..b5856a65 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala @@ -16,6 +16,7 @@ package app.softnetwork.elastic.sql.bridge +import app.softnetwork.elastic.sql.PainlessContextType import app.softnetwork.elastic.sql.operator.AND import app.softnetwork.elastic.sql.query.{ BetweenExpr, @@ -45,7 +46,10 @@ case class ElasticBridge(filter: ElasticFilter) { def query( innerHitsNames: Set[String] = Set.empty, currentQuery: Option[ElasticBoolQuery] - )(implicit timestamp: Long): Query = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Query = { filter match { case boolQuery: ElasticBoolQuery => import boolQuery._ diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala index f745889a..efb14be1 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala @@ -16,13 +16,15 @@ package app.softnetwork.elastic.sql.bridge +import app.softnetwork.elastic.sql.PainlessContextType import app.softnetwork.elastic.sql.query.Criteria import com.sksamuel.elastic4s.searches.queries.Query case class ElasticCriteria(criteria: Criteria) { def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty)(implicit - timestamp: Long + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query ): Query = { val query = criteria.boolQuery.copy(group = group) query diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 99e37c2c..1a1b2ad6 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -23,10 +23,14 @@ import app.softnetwork.elastic.sql.`type`.{ SQLTemporal, SQLVarchar } +import app.softnetwork.elastic.sql.config.ElasticSqlConfig import app.softnetwork.elastic.sql.function.aggregate.COUNT import app.softnetwork.elastic.sql.function.geo.{Distance, Meters} import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql.query._ +import app.softnetwork.elastic.sql.schema.mapper +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.NullNode import com.sksamuel.elastic4s.{ElasticApi, FetchSourceContext} import com.sksamuel.elastic4s.ElasticApi._ import com.sksamuel.elastic4s.http.ElasticDsl.BuildableTermsNoOp @@ -44,20 +48,34 @@ import com.sksamuel.elastic4s.searches.{MultiSearchRequest, SearchRequest} import com.sksamuel.elastic4s.searches.sort.{FieldSort, ScriptSort, ScriptSortType} import scala.language.implicitConversions +import scala.util.{Failure, Success, Try} package object bridge { - def now(script: Script)(implicit timestamp: Long): Script = { + lazy val sqlConfig: ElasticSqlConfig = ElasticSqlConfig() + + def now(script: Script)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Script = { if (!script.script.contains("params.__now__")) { return script } - script.param("__now__", timestamp) + contextType match { + case PainlessContextType.Query => script.param("__now__", timestamp) + case PainlessContextType.Transform => + script.param("__now__", sqlConfig.transformLastUpdatedColumnName) + case _ => script + } } implicit def requestToNestedFilterAggregation( request: SingleSearch, innerHitsName: String - )(implicit timestamp: Long): Option[FilterAggregation] = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Option[FilterAggregation] = { val having: Option[Query] = request.having.flatMap(_.criteria) match { case Some(f) => @@ -133,7 +151,10 @@ package object bridge { implicit def requestToFilterAggregation( request: SingleSearch - )(implicit timestamp: Long): Option[FilterAggregation] = + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Option[FilterAggregation] = request.having.flatMap(_.criteria) match { case Some(f) => val boolQuery = Option(ElasticBoolQuery(group = true)) @@ -151,7 +172,10 @@ package object bridge { implicit def requestToRootAggregations( request: SingleSearch, aggregations: Seq[ElasticAggregation] - )(implicit timestamp: Long): Seq[AbstractAggregation] = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Seq[AbstractAggregation] = { val notNestedAggregations = aggregations.filterNot(_.nested) val notNestedBuckets = request.bucketTree.filterNot(_.bucket.nested) @@ -201,7 +225,10 @@ package object bridge { implicit def requestToScopedAggregations( request: SingleSearch, aggregations: Seq[ElasticAggregation] - )(implicit timestamp: Long): Seq[NestedAggregation] = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Seq[NestedAggregation] = { // Group nested aggregations by their nested path val nestedAggregations: Map[String, Seq[ElasticAggregation]] = aggregations .filter(_.nested) @@ -407,7 +434,8 @@ package object bridge { } implicit def requestToElasticSearchRequest(request: SingleSearch)(implicit - timestamp: Long + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query ): ElasticSearchRequest = ElasticSearchRequest( request.sql, @@ -425,7 +453,10 @@ package object bridge { implicit def requestToSearchRequest( request: SingleSearch - )(implicit timestamp: Long): SearchRequest = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): SearchRequest = { import request._ val aggregations = request.aggregates.map( @@ -485,7 +516,7 @@ package object bridge { case Nil => _search case _ => _search scriptfields scriptFields.map { field => - val context = PainlessContext() + val context = PainlessContext(context = contextType) val script = field.painless(Some(context)) scriptField( field.scriptName, @@ -506,7 +537,7 @@ package object bridge { case Some(o) if aggregates.isEmpty && buckets.isEmpty => _search sortBy o.sorts.map { sort => if (sort.isScriptSort) { - val context = PainlessContext() + val context = PainlessContext(context = contextType) val painless = sort.field.painless(Some(context)) val painlessScript = s"$context$painless" val script = @@ -565,7 +596,10 @@ package object bridge { implicit def requestToMultiSearchRequest( request: MultiSearch - )(implicit timestamp: Long): MultiSearchRequest = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): MultiSearchRequest = { MultiSearchRequest( request.requests.map(implicitly[SearchRequest](_)) ) @@ -576,7 +610,10 @@ package object bridge { doubleOp: Double => A ): A = n.toEither.fold(longOp, doubleOp) - implicit def expressionToQuery(expression: GenericExpression)(implicit timestamp: Long): Query = { + implicit def expressionToQuery(expression: GenericExpression)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Query = { import expression._ if (isAggregation) return matchAllQuery() @@ -586,7 +623,7 @@ package object bridge { case _ => true })) ) { - val context = PainlessContext() + val context = PainlessContext(context = contextType) val script = painless(Some(context)) return scriptQuery( now( @@ -808,7 +845,7 @@ package object bridge { case NE | DIFF => not(rangeQuery(identifier.name) gte script lte script) } case _ => - val context = PainlessContext() + val context = PainlessContext(context = contextType) val script = painless(Some(context)) scriptQuery( now( @@ -819,7 +856,7 @@ package object bridge { ) } case _ => - val context = PainlessContext() + val context = PainlessContext(context = contextType) val script = painless(Some(context)) scriptQuery( now( @@ -882,7 +919,10 @@ package object bridge { implicit def betweenToQuery( between: BetweenExpr - )(implicit timestamp: Long): Query = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Query = { import between._ // Geo distance special case identifier.functions.headOption match { @@ -1005,6 +1045,44 @@ package object bridge { ) } + implicit def queryToString( + query: Query + ): String = { + SearchBodyBuilderFn( + ElasticApi.search("") query { + query + } + ).string() + } + + implicit def queryToJson( + query: Query + ): JsonNode = { + Try(mapper.readTree(queryToString(query))) match { + case Success(node) => + if (node.has("query")) { + node.get("query") + } else { + node + } + case Failure(_) => NullNode.instance + } + } + + implicit def criteriaToQuery(criteria: Criteria)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Query = { + ElasticCriteria(criteria).asQuery() + } + + implicit def criteriaToNode(criteria: Criteria)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): JsonNode = { + queryToJson(criteriaToQuery(criteria)) + } + implicit def filterToQuery( filter: ElasticFilter ): ElasticBridge = { @@ -1014,7 +1092,10 @@ package object bridge { @deprecated implicit def sqlQueryToAggregations( query: SelectStatement - )(implicit timestamp: Long): Seq[ElasticAggregation] = { + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): Seq[ElasticAggregation] = { import query._ statement .map { diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala index 9a31eaa1..bf607664 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala @@ -46,6 +46,8 @@ trait JestClientApi with JestVersionApi with JestPipelineApi with JestTemplateApi + with JestEnrichPolicyApi + with JestTransformApi with JestClientCompanion object JestClientApi extends SerializationApi { diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestEnrichPolicyApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestEnrichPolicyApi.scala new file mode 100644 index 00000000..e1364edf --- /dev/null +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestEnrichPolicyApi.scala @@ -0,0 +1,58 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.jest + +import app.softnetwork.elastic.client.result.ElasticFailure +import app.softnetwork.elastic.client.{result, EnrichPolicyApi} +import app.softnetwork.elastic.sql.schema + +trait JestEnrichPolicyApi extends EnrichPolicyApi with JestClientHelpers { + _: JestVersionApi with JestClientCompanion => + + override private[client] def executeCreateEnrichPolicy( + policy: schema.EnrichPolicy + ): result.ElasticResult[Boolean] = + ElasticFailure( + result.ElasticError( + message = "Enrich policy creation not implemented for Jest client", + operation = Some("CreateEnrichPolicy"), + statusCode = Some(501) + ) + ) + + override private[client] def executeDeleteEnrichPolicy( + policyName: String + ): result.ElasticResult[Boolean] = + ElasticFailure( + result.ElasticError( + message = "Enrich policy deletion not implemented for Jest client", + operation = Some("CreateEnrichPolicy"), + statusCode = Some(501) + ) + ) + + override private[client] def executeExecuteEnrichPolicy( + policyName: String + ): result.ElasticResult[String] = + ElasticFailure( + result.ElasticError( + message = "Enrich policy execution not implemented for Jest client", + operation = Some("ExecuteEnrichPolicy"), + statusCode = Some(501) + ) + ) +} diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala index fe53c9c3..a55e9580 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala @@ -102,4 +102,29 @@ trait JestMappingApi extends MappingApi with JestClientHelpers { } } } + + override private[client] def executeGetAllMappings(): ElasticResult[Map[String, String]] = + executeJestAction( + operation = "getAllMappings", + index = None, + retryable = true + )( + new GetMapping.Builder().build() + ) { result => + val jsonString = result.getJsonString + val jsonObject = JsonParser.parseString(jsonString).getAsJsonObject + val mappings = jsonObject + .entrySet() + .toArray + .map { entry => + val e = entry.asInstanceOf[java.util.Map.Entry[String, com.google.gson.JsonElement]] + val indexName = e.getKey + val mappingObj = e.getValue.getAsJsonObject + .getAsJsonObject("mappings") + .getAsJsonObject("_doc") + indexName -> mappingObj.toString + } + .toMap + mappings + } } diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala index 21809931..f69758e9 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala @@ -18,6 +18,7 @@ package app.softnetwork.elastic.client.jest import app.softnetwork.elastic.client.{ElasticQueries, ElasticQuery, SearchApi, SerializationApi} import app.softnetwork.elastic.client.result.ElasticResult +import app.softnetwork.elastic.sql.PainlessContextType import app.softnetwork.elastic.sql.bridge.ElasticSearchRequest import app.softnetwork.elastic.sql.query.SingleSearch import io.searchbox.core.MultiSearch @@ -29,9 +30,12 @@ import scala.language.implicitConversions trait JestSearchApi extends SearchApi with JestClientHelpers { _: JestClientCompanion with SerializationApi => - private[client] implicit def sqlSearchRequestToJsonQuery( + private[client] implicit def singleSearchToJsonQuery( sqlSearch: SingleSearch - )(implicit timestamp: Long): String = + )(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): String = implicitly[ElasticSearchRequest](sqlSearch).query import JestClientApi._ diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTransformApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTransformApi.scala new file mode 100644 index 00000000..c76855df --- /dev/null +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTransformApi.scala @@ -0,0 +1,83 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.jest + +import app.softnetwork.elastic.client.{result, TransformApi} +import app.softnetwork.elastic.sql.schema + +trait JestTransformApi extends TransformApi with JestClientHelpers { + _: JestClientCompanion => + + override private[client] def executeCreateTransform( + config: schema.TransformConfig, + start: Boolean + ): result.ElasticResult[Boolean] = + result.ElasticFailure( + result.ElasticError( + message = "Transform creation not implemented for Jest client", + operation = Some("CreateTransform"), + statusCode = Some(501) + ) + ) + + override private[client] def executeDeleteTransform( + transformId: String, + force: Boolean + ): result.ElasticResult[Boolean] = + result.ElasticFailure( + result.ElasticError( + message = "Transform deletion not implemented for Jest client", + operation = Some("DeleteTransform"), + statusCode = Some(501) + ) + ) + + override private[client] def executeStartTransform( + transformId: String + ): result.ElasticResult[Boolean] = + result.ElasticFailure( + result.ElasticError( + message = "Transform start not implemented for Jest client", + operation = Some("StartTransform"), + statusCode = Some(501) + ) + ) + + override private[client] def executeStopTransform( + transformId: String, + force: Boolean, + waitForCompletion: Boolean + ): result.ElasticResult[Boolean] = + result.ElasticFailure( + result.ElasticError( + message = "Transform stop not implemented for Jest client", + operation = Some("StopTransform"), + statusCode = Some(501) + ) + ) + + override private[client] def executeGetTransformStats( + transformId: String + ): result.ElasticResult[Option[schema.TransformStats]] = + result.ElasticFailure( + result.ElasticError( + message = "Get Transform stats not implemented for Jest client", + operation = Some("GetTransformStats"), + statusCode = Some(501) + ) + ) +} diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 66dc032f..b858aaec 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -21,12 +21,17 @@ import akka.actor.ActorSystem import akka.stream.scaladsl.{Flow, Source} import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ -import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} +import app.softnetwork.elastic.client.result.{ + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess +} import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.sql.{ObjectValue, Value} +import app.softnetwork.elastic.sql.{schema, ObjectValue, PainlessContextType, Value} import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.schema.TableAlias +import app.softnetwork.elastic.sql.schema.{EnrichPolicy, TableAlias} import com.fasterxml.jackson.databind.node.ObjectNode import com.google.gson.JsonParser import org.apache.http.util.EntityUtils @@ -114,6 +119,8 @@ trait RestHighLevelClientApi with RestHighLevelClientVersionApi with RestHighLevelClientPipelineApi with RestHighLevelClientTemplateApi + with RestHighLevelClientEnrichPolicyApi + with RestHighLevelClientTransformApi /** Version API implementation for RestHighLevelClient * @see @@ -608,6 +615,28 @@ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientH } }) + override private[client] def executeGetAllMappings(): ElasticResult[Map[String, String]] = + executeRestAction[ + GetMappingsRequest, + GetMappingsResponse, + Map[String, String] + ]( + operation = "getAllMappings", + index = None, + retryable = true + )( + request = new GetMappingsRequest().indices() + )( + executor = req => apply().indices().getMapping(req, RequestOptions.DEFAULT) + )(response => { + response + .mappings() + .asScala + .map { case (index, metadata) => + (index, metadata.source().toString) + } + .toMap + }) } /** Refresh API implementation for RestHighLevelClient @@ -942,8 +971,9 @@ trait RestHighLevelClientGetApi extends GetApi with RestHighLevelClientHelpers { trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHelpers { _: ElasticConversion with RestHighLevelClientCompanion with SerializationApi => - override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit - timestamp: Long + override implicit def singleSearchToJsonQuery(sqlSearch: SingleSearch)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query ): String = implicitly[ElasticSearchRequest](sqlSearch).query @@ -1899,3 +1929,106 @@ trait RestHighLevelClientTemplateApi extends TemplateApi with RestHighLevelClien } } } + +// ==================== ENRICH POLICY API IMPLEMENTATION FOR REST HIGH LEVEL CLIENT ==================== +trait RestHighLevelClientEnrichPolicyApi extends EnrichPolicyApi with RestHighLevelClientHelpers { + _: RestHighLevelClientVersionApi with RestHighLevelClientCompanion => + + override private[client] def executeCreateEnrichPolicy( + policy: EnrichPolicy + ): result.ElasticResult[Boolean] = + ElasticFailure( + result.ElasticError( + message = "Enrich policy creation not implemented for Rest client", + operation = Some("CreateEnrichPolicy"), + statusCode = Some(501) + ) + ) + + override private[client] def executeDeleteEnrichPolicy( + policyName: String + ): result.ElasticResult[Boolean] = + ElasticFailure( + result.ElasticError( + message = "Enrich policy deletion not implemented for Rest client", + operation = Some("CreateEnrichPolicy"), + statusCode = Some(501) + ) + ) + + override private[client] def executeExecuteEnrichPolicy( + policyName: String + ): ElasticResult[String] = + ElasticFailure( + result.ElasticError( + message = "Enrich policy execution not implemented for Rest client", + operation = Some("ExecuteEnrichPolicy"), + statusCode = Some(501) + ) + ) +} + +// ==================== TRANSFORM API IMPLEMENTATION FOR REST HIGH LEVEL CLIENT ==================== + +trait RestHighLevelClientTransformApi extends TransformApi with RestHighLevelClientHelpers { + _: RestHighLevelClientVersionApi with RestHighLevelClientCompanion with SerializationApi => + + override private[client] def executeCreateTransform( + config: schema.TransformConfig, + start: Boolean + ): result.ElasticResult[Boolean] = + result.ElasticFailure( + result.ElasticError( + message = "Transform creation not implemented for Rest client", + operation = Some("CreateTransform"), + statusCode = Some(501) + ) + ) + + override private[client] def executeDeleteTransform( + transformId: String, + force: Boolean + ): result.ElasticResult[Boolean] = + result.ElasticFailure( + result.ElasticError( + message = "Transform deletion not implemented for Rest client", + operation = Some("DeleteTransform"), + statusCode = Some(501) + ) + ) + + override private[client] def executeStartTransform( + transformId: String + ): result.ElasticResult[Boolean] = + result.ElasticFailure( + result.ElasticError( + message = "Transform start not implemented for Rest client", + operation = Some("StartTransform"), + statusCode = Some(501) + ) + ) + + override private[client] def executeStopTransform( + transformId: String, + force: Boolean, + waitForCompletion: Boolean + ): result.ElasticResult[Boolean] = + result.ElasticFailure( + result.ElasticError( + message = "Transform stop not implemented for Rest client", + operation = Some("StopTransform"), + statusCode = Some(501) + ) + ) + + override private[client] def executeGetTransformStats( + transformId: String + ): result.ElasticResult[Option[schema.TransformStats]] = + result.ElasticFailure( + result.ElasticError( + message = "Get Transform stats not implemented for Rest client", + operation = Some("GetTransformStats"), + statusCode = Some(501) + ) + ) +} diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index c21aaf84..78aaba78 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -23,11 +23,24 @@ import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.sql.{ObjectValue, Value} +import app.softnetwork.elastic.sql.{schema, ObjectValue, PainlessContextType, Value} import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} -import app.softnetwork.elastic.sql.schema.TableAlias +import app.softnetwork.elastic.sql.query.{Asc, Criteria, Desc, SQLAggregation, SingleSearch} +import app.softnetwork.elastic.sql.schema.{ + AvgTransformAggregation, + CardinalityTransformAggregation, + CountTransformAggregation, + EnrichPolicy, + MaxTransformAggregation, + MinTransformAggregation, + SumTransformAggregation, + TableAlias, + TermsGroupBy, + TopHitsTransformAggregation, + TransformState +} import app.softnetwork.elastic.sql.serialization.JacksonConfig +import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.ObjectNode import com.google.gson.JsonParser import org.apache.http.util.EntityUtils @@ -68,6 +81,7 @@ import org.elasticsearch.action.update.{UpdateRequest, UpdateResponse} import org.elasticsearch.action.{ActionListener, DocWriteRequest, DocWriteResponse} import org.elasticsearch.client.{GetAliasesResponse, Request, RequestOptions, Response} import org.elasticsearch.client.core.{CountRequest, CountResponse} +import org.elasticsearch.client.enrich.{DeletePolicyRequest, ExecutePolicyRequest, PutPolicyRequest} import org.elasticsearch.client.indices.{ CloseIndexRequest, ComposableIndexTemplateExistRequest, @@ -84,12 +98,45 @@ import org.elasticsearch.client.indices.{ PutIndexTemplateRequest, PutMappingRequest } +import org.elasticsearch.client.transform.transforms.latest.LatestConfig +import org.elasticsearch.client.transform.{ + DeleteTransformRequest, + GetTransformStatsRequest, + PutTransformRequest, + StartTransformRequest, + StopTransformRequest +} +import org.elasticsearch.client.transform.transforms.pivot.{ + AggregationConfig, + GroupConfig, + PivotConfig, + TermsGroupSource +} +import org.elasticsearch.client.transform.transforms.{ + DestConfig, + QueryConfig, + SourceConfig, + TimeSyncConfig, + TransformConfig +} import org.elasticsearch.cluster.metadata.{AliasMetadata, ComposableIndexTemplate} import org.elasticsearch.common.Strings import org.elasticsearch.common.bytes.BytesArray import org.elasticsearch.core.TimeValue +import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.xcontent.{DeprecationHandler, ToXContent, XContentFactory, XContentType} import org.elasticsearch.rest.RestStatus +import org.elasticsearch.search.aggregations.AggregatorFactories +import org.elasticsearch.search.aggregations.metrics.{ + AvgAggregationBuilder, + CardinalityAggregationBuilder, + MaxAggregationBuilder, + MinAggregationBuilder, + SumAggregationBuilder, + TopHitsAggregationBuilder, + ValueCountAggregationBuilder +} +import org.elasticsearch.search.aggregations.pipeline.BucketSelectorPipelineAggregationBuilder import org.elasticsearch.search.builder.{PointInTimeBuilder, SearchSourceBuilder} import org.elasticsearch.search.sort.{FieldSortBuilder, SortOrder} import org.json4s.jackson.JsonMethods @@ -121,6 +168,8 @@ trait RestHighLevelClientApi with RestHighLevelClientVersionApi with RestHighLevelClientPipelineApi with RestHighLevelClientTemplateApi + with RestHighLevelClientEnrichPolicyApi + with RestHighLevelClientTransformApi /** Version API implementation for RestHighLevelClient * @see @@ -597,6 +646,28 @@ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientH } }) + override private[client] def executeGetAllMappings(): ElasticResult[Map[String, String]] = + executeRestAction[ + GetMappingsRequest, + org.elasticsearch.client.indices.GetMappingsResponse, + Map[String, String] + ]( + operation = "getAllMappings", + index = None, + retryable = true + )( + request = new GetMappingsRequest().indices() + )( + executor = req => apply().indices().getMapping(req, RequestOptions.DEFAULT) + )(response => { + response + .mappings() + .asScala + .map { case (index, mappings) => + (index, mappings.source().toString) + } + .toMap + }) } /** Refresh API implementation for RestHighLevelClient @@ -930,10 +1001,11 @@ trait RestHighLevelClientGetApi extends GetApi with RestHighLevelClientHelpers { trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHelpers { _: ElasticConversion with RestHighLevelClientCompanion with SerializationApi => - override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit - timestamp: Long + override implicit def singleSearchToJsonQuery(singleSearch: SingleSearch)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query ): String = - implicitly[ElasticSearchRequest](sqlSearch).query + implicitly[ElasticSearchRequest](singleSearch).query override private[client] def executeSingleSearch( elasticQuery: ElasticQuery @@ -2166,3 +2238,314 @@ trait RestHighLevelClientTemplateApi extends TemplateApi with RestHighLevelClien } } } + +// ==================== ENRICH POLICY API IMPLEMENTATION FOR REST HIGH LEVEL CLIENT ==================== + +trait RestHighLevelClientEnrichPolicyApi extends EnrichPolicyApi with RestHighLevelClientHelpers { + _: RestHighLevelClientVersionApi with RestHighLevelClientCompanion => + + override private[client] def executeCreateEnrichPolicy( + policy: EnrichPolicy + ): result.ElasticResult[Boolean] = + executeRestAction( + operation = "CreateEnrichPolicy", + retryable = false + )( + request = { + new PutPolicyRequest( + policy.name, + policy.policyType.name.toLowerCase, + policy.indices.asJava, + policy.matchField, + policy.enrichFields.asJava + ) + } + )( + executor = req => apply().enrich().putPolicy(req, RequestOptions.DEFAULT) + )( + transformer = resp => resp.isAcknowledged + ) + + override private[client] def executeDeleteEnrichPolicy( + policyName: String + ): result.ElasticResult[Boolean] = + executeRestAction( + operation = "DeleteEnrichPolicy", + retryable = false + )( + request = new DeletePolicyRequest(policyName) + )( + executor = req => apply().enrich().deletePolicy(req, RequestOptions.DEFAULT) + )( + transformer = resp => resp.isAcknowledged + ) + + override private[client] def executeExecuteEnrichPolicy( + policyName: String + ): ElasticResult[String] = + executeRestAction( + operation = "ExecuteEnrichPolicy", + retryable = false + )( + request = { + val req = new ExecutePolicyRequest(policyName) + req.setWaitForCompletion(true) + req + } + )( + executor = req => apply().enrich().executePolicy(req, RequestOptions.DEFAULT) + )( + transformer = resp => resp.getTaskId + ) +} + +// ==================== TRANSFORM API IMPLEMENTATION FOR REST HIGH LEVEL CLIENT ==================== + +trait RestHighLevelClientTransformApi extends TransformApi with RestHighLevelClientHelpers { + _: RestHighLevelClientVersionApi with RestHighLevelClientCompanion with SerializationApi => + + override private[client] def executeCreateTransform( + config: schema.TransformConfig, + start: Boolean + ): result.ElasticResult[Boolean] = { + executeRestAction( + operation = "createTransform", + retryable = false + )( + request = { + implicit val timestamp: Long = System.currentTimeMillis() + implicit val context: PainlessContextType = PainlessContextType.Transform + val conf = convertToElasticTransformConfig(config) + val req = new PutTransformRequest(conf) + req.setDeferValidation(false) + req + } + )( + executor = req => apply().transform().putTransform(req, RequestOptions.DEFAULT) + )( + transformer = resp => resp.isAcknowledged + ) + } + + private def convertToElasticTransformConfig( + config: schema.TransformConfig + )(implicit criteriaToNode: Criteria => JsonNode): TransformConfig = { + val builder = TransformConfig.builder() + + // Set ID + builder.setId(config.id) + + // Set description + builder.setDescription(config.description) + + // Set source + val sourceConfig = SourceConfig + .builder() + .setIndex(config.source.index: _*) + + config.source.query.foreach { criteria => + // Convert Criteria to QueryBuilder + val node: JsonNode = criteria + val queryJson = mapper.writeValueAsString(node) + sourceConfig.setQueryConfig( + new QueryConfig(QueryBuilders.wrapperQuery(queryJson)) + ) + } + builder.setSource(sourceConfig.build()) + + // Set destination + val destConfig = DestConfig + .builder() + .setIndex(config.dest.index) + config.dest.pipeline.foreach(destConfig.setPipeline) + builder.setDest(destConfig.build()) + + // Set frequency + builder.setFrequency( + org.elasticsearch.core.TimeValue.parseTimeValue( + config.frequency.toTransformFormat, + "frequency" + ) + ) + + // Set sync (if present) + config.sync.foreach { sync => + val syncConfig = TimeSyncConfig + .builder() + .setField(sync.time.field) + .setDelay( + org.elasticsearch.core.TimeValue.parseTimeValue( + sync.time.delay.toTransformFormat, + "delay" + ) + ) + .build() + builder.setSyncConfig(syncConfig) + } + + // Set pivot (if present) + config.pivot.foreach { pivot => + val pivotConfig = PivotConfig.builder() + + // Group by + val groupConfig = GroupConfig.builder() + pivot.groupBy.foreach { case (name, gb) => + gb match { + case TermsGroupBy(field) => + groupConfig + .groupBy(name, TermsGroupSource.builder().setField(field).build()) + } + } + pivotConfig.setGroups(groupConfig.build()) + + // Aggregations + val aggBuilder = AggregatorFactories.builder() + pivot.aggregations.foreach { case (name, agg) => + agg match { + case MaxTransformAggregation(field) => + aggBuilder.addAggregator(new MaxAggregationBuilder(name).field(field)) + case MinTransformAggregation(field) => + aggBuilder.addAggregator(new MinAggregationBuilder(name).field(field)) + case SumTransformAggregation(field) => + aggBuilder.addAggregator(new SumAggregationBuilder(name).field(field)) + case AvgTransformAggregation(field) => + aggBuilder.addAggregator(new AvgAggregationBuilder(name).field(field)) + case CountTransformAggregation(field) => + aggBuilder.addAggregator(new ValueCountAggregationBuilder(name).field(field)) + case CardinalityTransformAggregation(field) => + aggBuilder.addAggregator(new CardinalityAggregationBuilder(name).field(field)) + case TopHitsTransformAggregation(fields, size, sortFields) => + val topHitsBuilder = new TopHitsAggregationBuilder(name) + .size(size) + .fetchSource(fields.toArray, Array.empty[String]) + sortFields.foreach { sortField => + val sortOrder = sortField.order.getOrElse(Desc) match { + case Asc => SortOrder.ASC + case Desc => SortOrder.DESC + case _ => SortOrder.DESC + } + topHitsBuilder.sort(sortField.name, sortOrder) + } + aggBuilder.addAggregator(topHitsBuilder) + case _ => + throw new UnsupportedOperationException(s"Unsupported aggregation: $agg") + } + } + pivot.bucketSelector foreach { bs => + val bucketSelector = new BucketSelectorPipelineAggregationBuilder( + bs.name, + bs.bucketsPath.asJava, + new org.elasticsearch.script.Script(bs.script) + ) + aggBuilder.addPipelineAggregator(bucketSelector) + } + pivotConfig.setAggregationConfig(new AggregationConfig(aggBuilder)) + builder.setPivotConfig(pivotConfig.build()) + } + + config.latest.foreach { latest => + val latestConfig = LatestConfig + .builder() + .setUniqueKey(latest.uniqueKey.asJava) + .setSort(latest.sort) + .build() + + builder.setLatestConfig(latestConfig) + } + + builder.setMetadata(config.metadata.asJava) + + builder.build() + } + + override private[client] def executeDeleteTransform( + transformId: String, + force: Boolean + ): result.ElasticResult[Boolean] = { + executeRestAction( + operation = "deleteTransform", + retryable = false + )( + request = { + val req = new DeleteTransformRequest(transformId) + req.setForce(force) + req + } + )( + executor = req => apply().transform().deleteTransform(req, RequestOptions.DEFAULT) + )( + transformer = resp => resp.isAcknowledged + ) + } + + override private[client] def executeStartTransform( + transformId: String + ): result.ElasticResult[Boolean] = { + executeRestAction( + operation = "startTransform", + retryable = false + )( + request = new StartTransformRequest(transformId) + )( + executor = req => apply().transform().startTransform(req, RequestOptions.DEFAULT) + )( + transformer = resp => resp.isAcknowledged + ) + } + + override private[client] def executeStopTransform( + transformId: String, + force: Boolean, + waitForCompletion: Boolean + ): result.ElasticResult[Boolean] = { + executeRestAction( + operation = "stopTransform", + retryable = false + )( + request = { + val req = new StopTransformRequest(transformId) + req.setWaitForCheckpoint(!force) + req.setWaitForCompletion(waitForCompletion) + req + } + )( + executor = req => apply().transform().stopTransform(req, RequestOptions.DEFAULT) + )( + transformer = resp => resp.isAcknowledged + ) + } + + override private[client] def executeGetTransformStats( + transformId: String + ): result.ElasticResult[Option[schema.TransformStats]] = { + executeRestAction( + operation = "getTransformStats", + retryable = true + )( + request = new GetTransformStatsRequest(transformId) + )( + executor = req => apply().transform().getTransformStats(req, RequestOptions.DEFAULT) + )( + transformer = resp => { + val statsArray = resp.getTransformsStats.asScala + statsArray.headOption.map { stats => + schema.TransformStats( + id = stats.getId, + state = TransformState(stats.getState.value()), + documentsProcessed = stats.getIndexerStats.getDocumentsProcessed, + documentsIndexed = stats.getIndexerStats.getDocumentsIndexed, + indexFailures = stats.getIndexerStats.getIndexFailures, + searchFailures = stats.getIndexerStats.getSearchFailures, + lastCheckpoint = Option(stats.getCheckpointingInfo) + .flatMap(c => Option(c.getLast)) + .map(_.getCheckpoint), + operationsBehind = Option(stats.getCheckpointingInfo) + .flatMap(info => Option(info.getOperationsBehind)) + .getOrElse(0L), + processingTimeMs = stats.getIndexerStats.getProcessingTime + ) + } + } + ) + } +} diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 9005dee0..b9c5862a 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -24,15 +24,16 @@ import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} +import app.softnetwork.elastic.sql.query.{Criteria, SQLAggregation, SingleSearch} +import app.softnetwork.elastic.sql.serialization._ import app.softnetwork.elastic.client.result.{ ElasticError, ElasticFailure, ElasticResult, ElasticSuccess } -import app.softnetwork.elastic.sql.schema.TableAlias -import app.softnetwork.elastic.sql.serialization._ +import app.softnetwork.elastic.sql.{schema, PainlessContextType} +import app.softnetwork.elastic.sql.schema.{EnrichPolicy, TableAlias, TransformState} import co.elastic.clients.elasticsearch._types.mapping.TypeMapping import co.elastic.clients.elasticsearch._types.{ Conflicts, @@ -60,6 +61,11 @@ import co.elastic.clients.elasticsearch.core.msearch.{ import co.elastic.clients.elasticsearch.core._ import co.elastic.clients.elasticsearch.core.reindex.{Destination, Source => ESSource} import co.elastic.clients.elasticsearch.core.search.PointInTimeReference +import co.elastic.clients.elasticsearch.enrich.{ + DeletePolicyRequest, + ExecutePolicyRequest, + PutPolicyRequest +} import co.elastic.clients.elasticsearch.indices.update_aliases.{Action, AddAction, RemoveAction} import co.elastic.clients.elasticsearch.indices.{ExistsRequest => IndexExistsRequest, _} import co.elastic.clients.elasticsearch.ingest.{ @@ -67,6 +73,13 @@ import co.elastic.clients.elasticsearch.ingest.{ GetPipelineRequest, PutPipelineRequest } +import co.elastic.clients.elasticsearch.transform.{ + DeleteTransformRequest, + GetTransformStatsRequest, + PutTransformRequest, + StartTransformRequest, + StopTransformRequest +} import com.fasterxml.jackson.databind.JsonNode import com.google.gson.JsonParser @@ -97,6 +110,8 @@ trait JavaClientApi with JavaClientVersionApi with JavaClientPipelineApi with JavaClientTemplateApi + with JavaClientEnrichPolicyApi + with JavaClientTransformApi /** Elasticsearch client implementation using the Java Client * @see @@ -558,6 +573,27 @@ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { } } } + + override private[client] def executeGetAllMappings(): ElasticResult[Map[String, String]] = + executeJavaAction( + operation = "getAllMappings", + index = None, + retryable = true + )( + apply() + .indices() + .getMapping( + new GetMappingRequest.Builder().build() + ) + ) { response => + response + .result() + .asScala + .map { case (index, mapping) => + (index, convertToJson(mapping)) + } + .toMap + } } /** Elasticsearch client implementation of Refresh API using the Java Client @@ -902,10 +938,11 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { _: JavaClientCompanion with SerializationApi => - override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit - timestamp: Long + override implicit def singleSearchToJsonQuery(singleSearch: SingleSearch)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query ): String = - implicitly[ElasticSearchRequest](sqlSearch).query + implicitly[ElasticSearchRequest](singleSearch).query override private[client] def executeSingleSearch( elasticQuery: ElasticQuery @@ -1950,3 +1987,188 @@ trait JavaClientTemplateApi extends TemplateApi with JavaClientHelpers { } } + +// ==================== ENRICH POLICY API IMPLEMENTATION FOR JAVA CLIENT ==================== + +trait JavaClientEnrichPolicyApi extends EnrichPolicyApi with JavaClientHelpers { + _: JavaClientVersionApi with JavaClientCompanion => + + override private[client] def executeCreateEnrichPolicy( + policy: EnrichPolicy + ): ElasticResult[Boolean] = { + implicit val timestamp: Long = System.currentTimeMillis() + executeJavaBooleanAction( + operation = "createEnrichPolicy", + index = None, + retryable = false + )( + apply() + .enrich() + .putPolicy( + new PutPolicyRequest.Builder() + .name(policy.name) + .withJson(new StringReader(policy.node)) + .build() + ) + )(resp => resp.acknowledged()) + } + + override private[client] def executeDeleteEnrichPolicy( + policyName: String + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "deleteEnrichPolicy", + index = None, + retryable = false + )( + apply() + .enrich() + .deletePolicy( + new DeletePolicyRequest.Builder() + .name(policyName) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeExecuteEnrichPolicy( + policyName: JSONQuery + ): ElasticResult[JSONQuery] = + executeJavaAction( + operation = "executeEnrichPolicy", + index = None, + retryable = false + )( + apply() + .enrich() + .executePolicy( + new ExecutePolicyRequest.Builder() + .name(policyName) + .waitForCompletion(true) + .build() + ) + )(resp => resp.task()) +} + +// ==================== TRANSFORM API IMPLEMENTATION FOR JAVA CLIENT ==================== + +trait JavaClientTransformApi extends TransformApi with JavaClientHelpers { + _: JavaClientVersionApi with JavaClientCompanion => + + override private[client] def executeCreateTransform( + config: schema.TransformConfig, + start: Boolean + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "createTransform", + index = None, + retryable = false + ) { + implicit val timestamp: Long = System.currentTimeMillis() + implicit val context: PainlessContextType = PainlessContextType.Transform + apply() + .transform() + .putTransform( + new PutTransformRequest.Builder() + .transformId(config.id) + .withJson(new StringReader(convertToElasticTransformConfig(config))) + .build() + ) + }(resp => resp.acknowledged()) + + private def convertToElasticTransformConfig( + config: schema.TransformConfig + )(implicit criteriaToNode: Criteria => JsonNode): String = { + config.node + } + + override private[client] def executeDeleteTransform( + transformId: String, + force: Boolean + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "deleteTransform", + index = None, + retryable = false + )( + apply() + .transform() + .deleteTransform( + new DeleteTransformRequest.Builder() + .transformId(transformId) + .force(force) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeStartTransform(transformId: String): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "startTransform", + index = None, + retryable = false + )( + apply() + .transform() + .startTransform( + new StartTransformRequest.Builder() + .transformId(transformId) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeStopTransform( + transformId: String, + force: Boolean, + waitForCompletion: Boolean + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "stopTransform", + index = None, + retryable = false + )( + apply() + .transform() + .stopTransform( + new StopTransformRequest.Builder() + .transformId(transformId) + .force(force) + .waitForCompletion(waitForCompletion) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeGetTransformStats( + transformId: String + ): ElasticResult[Option[schema.TransformStats]] = + executeJavaAction( + operation = "getTransformStats", + index = None, + retryable = true + )( + apply() + .transform() + .getTransformStats( + new GetTransformStatsRequest.Builder() + .transformId(transformId) + .build() + ) + ) { resp => + val statsOpt = resp.transforms().asScala.headOption.map { stats => + schema.TransformStats( + id = stats.id(), + state = TransformState(stats.state()), + documentsProcessed = stats.stats().documentsProcessed(), + documentsIndexed = stats.stats().documentsIndexed(), + indexFailures = stats.stats().indexFailures(), + searchFailures = stats.stats().searchFailures(), + lastCheckpoint = Option(stats.checkpointing()) + .flatMap(c => Option(c.last())) + .map(_.checkpoint()), + operationsBehind = Option(stats.checkpointing()) + .flatMap(info => Option(info.operationsBehind().longValue())) + .getOrElse(0L), + processingTimeMs = stats.stats().processingTimeInMs() + ) + } + statsOpt + } +} diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 563120b2..3a612ece 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -24,14 +24,15 @@ import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} +import app.softnetwork.elastic.sql.query.{Criteria, SQLAggregation, SingleSearch} import app.softnetwork.elastic.client.result.{ ElasticError, ElasticFailure, ElasticResult, ElasticSuccess } -import app.softnetwork.elastic.sql.schema.TableAlias +import app.softnetwork.elastic.sql.{schema, PainlessContextType} +import app.softnetwork.elastic.sql.schema.{EnrichPolicy, TableAlias, TransformState} import app.softnetwork.elastic.sql.serialization._ import co.elastic.clients.elasticsearch._types.mapping.TypeMapping import co.elastic.clients.elasticsearch._types.{ @@ -55,6 +56,11 @@ import co.elastic.clients.elasticsearch.core.msearch.{MultisearchHeader, Request import co.elastic.clients.elasticsearch.core._ import co.elastic.clients.elasticsearch.core.reindex.{Destination, Source => ESSource} import co.elastic.clients.elasticsearch.core.search.{PointInTimeReference, SearchRequestBody} +import co.elastic.clients.elasticsearch.enrich.{ + DeletePolicyRequest, + ExecutePolicyRequest, + PutPolicyRequest +} import co.elastic.clients.elasticsearch.indices.update_aliases.{Action, AddAction, RemoveAction} import co.elastic.clients.elasticsearch.indices.{ExistsRequest => IndexExistsRequest, _} import co.elastic.clients.elasticsearch.ingest.{ @@ -62,6 +68,13 @@ import co.elastic.clients.elasticsearch.ingest.{ GetPipelineRequest, PutPipelineRequest } +import co.elastic.clients.elasticsearch.transform.{ + DeleteTransformRequest, + GetTransformStatsRequest, + PutTransformRequest, + StartTransformRequest, + StopTransformRequest +} import com.fasterxml.jackson.databind.JsonNode import com.google.gson.JsonParser @@ -92,6 +105,8 @@ trait JavaClientApi with JavaClientVersionApi with JavaClientPipelineApi with JavaClientTemplateApi + with JavaClientEnrichPolicyApi + with JavaClientTransformApi /** Elasticsearch client implementation using the Java Client * @see @@ -558,6 +573,26 @@ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { } } + override private[client] def executeGetAllMappings(): ElasticResult[Map[String, String]] = + executeJavaAction( + operation = "getAllMappings", + index = None, + retryable = true + )( + apply() + .indices() + .getMapping( + new GetMappingRequest.Builder().build() + ) + ) { response => + response + .mappings() + .asScala + .map { case (index, mapping) => + (index, convertToJson(mapping)) + } + .toMap + } } /** Elasticsearch client implementation of Refresh API using the Java Client @@ -899,10 +934,11 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { _: JavaClientCompanion with SerializationApi => - override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit - timestamp: Long + override implicit def singleSearchToJsonQuery(singleSearch: SingleSearch)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query ): String = - implicitly[ElasticSearchRequest](sqlSearch).query + implicitly[ElasticSearchRequest](singleSearch).query override private[client] def executeSingleSearch( elasticQuery: ElasticQuery @@ -1947,3 +1983,188 @@ trait JavaClientTemplateApi extends TemplateApi with JavaClientHelpers with Java } } + +// ==================== ENRICH POLICY API IMPLEMENTATION FOR JAVA CLIENT ==================== + +trait JavaClientEnrichPolicyApi extends EnrichPolicyApi with JavaClientHelpers { + _: JavaClientVersionApi with JavaClientCompanion => + + override private[client] def executeCreateEnrichPolicy( + policy: EnrichPolicy + ): ElasticResult[Boolean] = { + implicit val timestamp: Long = System.currentTimeMillis() + executeJavaBooleanAction( + operation = "createEnrichPolicy", + index = None, + retryable = false + )( + apply() + .enrich() + .putPolicy( + new PutPolicyRequest.Builder() + .name(policy.name) + .withJson(new StringReader(policy.node)) + .build() + ) + )(resp => resp.acknowledged()) + } + + override private[client] def executeDeleteEnrichPolicy( + policyName: String + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "deleteEnrichPolicy", + index = None, + retryable = false + )( + apply() + .enrich() + .deletePolicy( + new DeletePolicyRequest.Builder() + .name(policyName) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeExecuteEnrichPolicy( + policyName: String + ): ElasticResult[String] = + executeJavaAction( + operation = "executeEnrichPolicy", + index = None, + retryable = false + )( + apply() + .enrich() + .executePolicy( + new ExecutePolicyRequest.Builder() + .name(policyName) + .waitForCompletion(true) + .build() + ) + )(resp => resp.task()) +} + +// ==================== TRANSFORM API IMPLEMENTATION FOR JAVA CLIENT ==================== + +trait JavaClientTransformApi extends TransformApi with JavaClientHelpers { + _: JavaClientVersionApi with JavaClientCompanion => + + override private[client] def executeCreateTransform( + config: schema.TransformConfig, + start: Boolean + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "createTransform", + index = None, + retryable = false + ) { + implicit val timestamp: Long = System.currentTimeMillis() + implicit val context: PainlessContextType = PainlessContextType.Transform + apply() + .transform() + .putTransform( + new PutTransformRequest.Builder() + .transformId(config.id) + .withJson(new StringReader(convertToElasticTransformConfig(config))) + .build() + ) + }(resp => resp.acknowledged()) + + private def convertToElasticTransformConfig( + config: schema.TransformConfig + )(implicit criteriaToNode: Criteria => JsonNode): String = { + config.node + } + + override private[client] def executeDeleteTransform( + transformId: String, + force: Boolean + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "deleteTransform", + index = None, + retryable = false + )( + apply() + .transform() + .deleteTransform( + new DeleteTransformRequest.Builder() + .transformId(transformId) + .force(force) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeStartTransform(transformId: String): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "startTransform", + index = None, + retryable = false + )( + apply() + .transform() + .startTransform( + new StartTransformRequest.Builder() + .transformId(transformId) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeStopTransform( + transformId: String, + force: Boolean, + waitForCompletion: Boolean + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "stopTransform", + index = None, + retryable = false + )( + apply() + .transform() + .stopTransform( + new StopTransformRequest.Builder() + .transformId(transformId) + .force(force) + .waitForCompletion(waitForCompletion) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeGetTransformStats( + transformId: String + ): ElasticResult[Option[schema.TransformStats]] = + executeJavaAction( + operation = "getTransformStats", + index = None, + retryable = true + )( + apply() + .transform() + .getTransformStats( + new GetTransformStatsRequest.Builder() + .transformId(transformId) + .build() + ) + ) { resp => + val statsOpt = resp.transforms().asScala.headOption.map { stats => + schema.TransformStats( + id = stats.id(), + state = TransformState(stats.state()), + documentsProcessed = stats.stats().documentsProcessed(), + documentsIndexed = stats.stats().documentsIndexed(), + indexFailures = stats.stats().indexFailures(), + searchFailures = stats.stats().searchFailures(), + lastCheckpoint = Option(stats.checkpointing()) + .flatMap(c => Option(c.last())) + .map(_.checkpoint()), + operationsBehind = Option(stats.checkpointing()) + .flatMap(info => Option(info.operationsBehind().longValue())) + .getOrElse(0L), + processingTimeMs = stats.stats().processingTimeInMs() + ) + } + statsOpt + } +} diff --git a/licensing/build.sbt b/licensing/build.sbt new file mode 100644 index 00000000..59ed8497 --- /dev/null +++ b/licensing/build.sbt @@ -0,0 +1,4 @@ +organization := "app.softnetwork.elastic" + +name := "softclient4es-licensing" + diff --git a/licensing/src/main/scala/app/softnetwork/elastic/licensing/DefaultLicenseManager.scala b/licensing/src/main/scala/app/softnetwork/elastic/licensing/DefaultLicenseManager.scala new file mode 100644 index 00000000..348086ab --- /dev/null +++ b/licensing/src/main/scala/app/softnetwork/elastic/licensing/DefaultLicenseManager.scala @@ -0,0 +1,78 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.licensing + +class DefaultLicenseManager extends LicenseManager { + + private var currentLicense: LicenseKey = LicenseKey( + id = "community", + licenseType = LicenseType.Community, + features = Set( + Feature.MaterializedViews, + Feature.JdbcDriver + ), + expiresAt = None + ) + + override def validate(key: String): Either[LicenseError, LicenseKey] = { + key match { + case k if k.startsWith("PRO-") => + val license = LicenseKey( + id = k, + licenseType = LicenseType.Pro, + features = Set( + Feature.MaterializedViews, + Feature.JdbcDriver, + Feature.UnlimitedResults + ), + expiresAt = None + ) + currentLicense = license + Right(license) + + case k if k.startsWith("ENT-") => + val license = LicenseKey( + id = k, + licenseType = LicenseType.Enterprise, + features = Set( + Feature.MaterializedViews, + Feature.JdbcDriver, + Feature.OdbcDriver, + Feature.UnlimitedResults + ), + expiresAt = None + ) + currentLicense = license + Right(license) + + case _ => + Left(InvalidLicense("Invalid license key format")) + } + } + + override def hasFeature(feature: Feature): Boolean = { + currentLicense.features.contains(feature) + } + + override def quotas: Quota = currentLicense.licenseType match { + case LicenseType.Community => Quota.Community + case LicenseType.Pro => Quota.Pro + case LicenseType.Enterprise => Quota.Enterprise + } + + override def licenseType: LicenseType = currentLicense.licenseType +} diff --git a/licensing/src/main/scala/app/softnetwork/elastic/licensing/LicenseChecker.scala b/licensing/src/main/scala/app/softnetwork/elastic/licensing/LicenseChecker.scala new file mode 100644 index 00000000..44dd33a3 --- /dev/null +++ b/licensing/src/main/scala/app/softnetwork/elastic/licensing/LicenseChecker.scala @@ -0,0 +1,60 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.licensing + +import scala.concurrent.Future + +/** Generic abstraction for license checking. + * + * Type parameters: + * - REQUEST: The request type to check (e.g., SQL statement, query) + * - RESULT: The result type to return (e.g., query result, DDL result) + */ +trait LicenseChecker[REQUEST, RESULT] { + + /** Check license for a request. + * + * @param request + * The request to check + * @param execute + * The actual execution function (if license allows) + * @return + * Either a license error or the execution result + */ + def check( + request: REQUEST, + execute: REQUEST => Future[RESULT] + ): Future[Either[LicenseError, RESULT]] +} + +/** Helper for creating checkers + */ +object LicenseChecker { + + /** No-op checker (always allows) + */ + def noOp[REQUEST, RESULT]: LicenseChecker[REQUEST, RESULT] = + new LicenseChecker[REQUEST, RESULT] { + override def check( + request: REQUEST, + execute: REQUEST => Future[RESULT] + ): Future[Either[LicenseError, RESULT]] = { + import scala.concurrent.ExecutionContext.Implicits.global + execute(request).map(Right(_)) + } + } +} diff --git a/licensing/src/main/scala/app/softnetwork/elastic/licensing/package.scala b/licensing/src/main/scala/app/softnetwork/elastic/licensing/package.scala new file mode 100644 index 00000000..e67c2caa --- /dev/null +++ b/licensing/src/main/scala/app/softnetwork/elastic/licensing/package.scala @@ -0,0 +1,125 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic + +package object licensing { + + sealed trait LicenseType { + def isPaid: Boolean = this != LicenseType.Community + def isEnterprise: Boolean = this == LicenseType.Enterprise + def isPro: Boolean = this == LicenseType.Pro + } + + object LicenseType { + case object Community extends LicenseType // Gratuit + case object Pro extends LicenseType // Payant + case object Enterprise extends LicenseType // Payant + support + def upgradeTo(licenseType: LicenseType): LicenseType = licenseType match { + case Community => Pro + case Pro => Enterprise + case Enterprise => Enterprise + } + } + + sealed trait Feature + + object Feature { + case object MaterializedViews extends Feature + case object JdbcDriver extends Feature + case object OdbcDriver extends Feature + case object UnlimitedResults extends Feature + case object AdvancedAggregations extends Feature + def values: Seq[Feature] = Seq( + MaterializedViews, + JdbcDriver, + OdbcDriver, + UnlimitedResults, + AdvancedAggregations + ) + } + + case class LicenseKey( + id: String, + licenseType: LicenseType, + features: Set[Feature], + expiresAt: Option[java.time.Instant], + metadata: Map[String, String] = Map.empty + ) + + case class Quota( + maxMaterializedViews: Option[Int], // None = unlimited + maxQueryResults: Option[Int], // None = unlimited + maxConcurrentQueries: Option[Int] + ) + + object Quota { + val Community: Quota = Quota( + maxMaterializedViews = Some(3), + maxQueryResults = Some(10000), + maxConcurrentQueries = Some(5) + ) + + val Pro: Quota = Quota( + maxMaterializedViews = Some(50), + maxQueryResults = Some(1000000), + maxConcurrentQueries = Some(50) + ) + + val Enterprise: Quota = Quota( + maxMaterializedViews = None, // Unlimited + maxQueryResults = None, + maxConcurrentQueries = None + ) + } + + trait LicenseManager { + + /** Validate license key */ + def validate(key: String): Either[LicenseError, LicenseKey] + + /** Check if feature is available */ + def hasFeature(feature: Feature): Boolean + + /** Get current quotas */ + def quotas: Quota + + /** Get license type */ + def licenseType: LicenseType + } + + sealed trait LicenseError { + def message: String + def statusCode: Int = 402 // Payment Required + } + + case class InvalidLicense(reason: String) extends LicenseError { + def message: String = s"Invalid license: $reason" + } + + case class ExpiredLicense(expiredAt: java.time.Instant) extends LicenseError { + def message: String = s"License expired at $expiredAt" + } + + case class FeatureNotAvailable(feature: Feature) extends LicenseError { + def message: String = s"Feature $feature requires a paid license" + } + + case class QuotaExceeded(quota: String, current: Int, max: Int) extends LicenseError { + def message: String = s"Quota exceeded: $quota ($current/$max)" + } + +} diff --git a/sql/src/main/resources/softnetwork-sql.conf b/sql/src/main/resources/softnetwork-sql.conf index 74fdf8e4..10c45b27 100644 --- a/sql/src/main/resources/softnetwork-sql.conf +++ b/sql/src/main/resources/softnetwork-sql.conf @@ -1,4 +1,5 @@ sql { composite-key-separator = "\\|\\|" # regex separator for composite keys in SQL ddl queries artificial-primary-key-column-name = "id" # name of the artificial primary key column added to tables without primary key(s) defined + transform-last-updated-column-name = "_last_updated" # name of the column used to track last updated timestamp for records } diff --git a/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala b/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala index 75c56386..92a5acb1 100644 --- a/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala +++ b/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala @@ -6,7 +6,8 @@ import configs.Configs case class ElasticSqlConfig( compositeKeySeparator: String, - artificialPrimaryKeyColumnName: String + artificialPrimaryKeyColumnName: String, + transformLastUpdatedColumnName: String ) object ElasticSqlConfig extends StrictLogging { diff --git a/sql/src/main/scala-2.13/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala b/sql/src/main/scala-2.13/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala index c777f29d..9fd99a49 100644 --- a/sql/src/main/scala-2.13/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala +++ b/sql/src/main/scala-2.13/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala @@ -22,7 +22,8 @@ import configs.ConfigReader case class ElasticSqlConfig( compositeKeySeparator: String, - artificialPrimaryKeyColumnName: String + artificialPrimaryKeyColumnName: String, + transformLastUpdatedColumnName: String ) object ElasticSqlConfig extends StrictLogging { diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index 304bda0c..3501fbaf 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -16,7 +16,14 @@ package app.softnetwork.elastic -import app.softnetwork.elastic.sql.{BooleanValue, ObjectValue, StringValue, StringValues, Value} +import app.softnetwork.elastic.sql.{ + BooleanValue, + ObjectValue, + ObjectValues, + StringValue, + StringValues, + Value +} import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.schema.{ Column, @@ -28,12 +35,14 @@ import app.softnetwork.elastic.sql.schema.{ Schema, ScriptProcessor, SetProcessor, - Table + Table, + TableType } import app.softnetwork.elastic.sql.serialization._ import app.softnetwork.elastic.sql.time.TimeUnit import com.fasterxml.jackson.databind.JsonNode +import java.security.MessageDigest import scala.jdk.CollectionConverters._ package object schema { @@ -46,7 +55,8 @@ package object schema { not_null: Option[Boolean] = None, comment: Option[String] = None, fields: List[IndexField] = Nil, - options: Map[String, Value[_]] = Map.empty + options: Map[String, Value[_]] = Map.empty, + lineage: Map[String, Seq[(String, String)]] = Map.empty // ✅ Added ) { lazy val ddlColumn: Column = { Column( @@ -57,7 +67,8 @@ package object schema { defaultValue = null_value, notNull = not_null.getOrElse(false), comment = comment, - options = options + options = options, + lineage = lineage // ✅ Added ) } } @@ -161,6 +172,46 @@ package object schema { } case _ => None } + + // ✅ Extract lineage from _meta + val lineage = _meta + .flatMap { + case m: ObjectValue => + m.value.get("lineage") match { + case Some(lin: ObjectValue) => + Some( + lin.value.flatMap { + case (pathId, pathValue: ObjectValues) => + // Parse the chain of (table, column) pairs + val chain = pathValue.values.flatMap { + case obj: ObjectValue => + val tableOpt = obj.value.get("table") match { + case Some(StringValue(t)) => Some(t) + case _ => None + } + val columnOpt = obj.value.get("column") match { + case Some(StringValue(c)) => Some(c) + case _ => None + } + for { + table <- tableOpt + column <- columnOpt + } yield (table, column) + case _ => None + } + + if (chain.nonEmpty) Some(pathId -> chain) + else None + + case _ => None + } + ) + case _ => None + } + case _ => None + } + .getOrElse(Map.empty) + IndexField( name = name, `type` = tpe, @@ -169,7 +220,8 @@ package object schema { not_null = notNull, comment = comment, fields = fields, - options = options + options = options, + lineage = lineage ) } @@ -179,10 +231,17 @@ package object schema { fields: List[IndexField] = Nil, primaryKey: List[String] = Nil, partitionBy: Option[IndexDatePartition] = None, - options: Map[String, Value[_]] = Map.empty + options: Map[String, Value[_]] = Map.empty, + materializedViews: Option[List[String]] = None, + tableType: TableType = TableType.Regular ) object IndexMappings { + def apply(json: String): IndexMappings = { + val root: JsonNode = json + apply(root) + } + def apply(root: JsonNode): IndexMappings = { if (root.has("mappings")) { val mappingsNode = root.path("mappings") @@ -248,11 +307,36 @@ package object schema { case _ => None } + val materializedViews: Option[List[String]] = meta + .map { + case m: ObjectValue => + m.value.get("materialized_views") match { + case Some(mvs: StringValues) => mvs.values.map(_.ddl.replaceAll("\"", "")).toList + case Some(mv: StringValue) => List(mv.ddl.replaceAll("\"", "")) + case _ => List.empty + } + case _ => List.empty + } + .filter(_.nonEmpty) + + val tableType: TableType = meta + .flatMap { + case m: ObjectValue => + m.value.get("type") match { + case Some(mv: StringValue) => Some(TableType(mv.value)) + case _ => None + } + case _ => None + } + .getOrElse(TableType.Regular) + IndexMappings( fields = fields, primaryKey = primaryKey, partitionBy = partitionBy, - options = options + options = options, + materializedViews = materializedViews, + tableType = tableType ) } @@ -433,7 +517,7 @@ package object schema { enrichedCols.update(col, c.copy(script = Some(p))) } - case p: SetProcessor => + case p: SetProcessor if p.isDefault => val col = p.column enrichedCols.get(col).foreach { c => enrichedCols.update(col, c.copy(defaultValue = Some(p.value))) @@ -465,7 +549,9 @@ package object schema { mappings = esMappings.options, settings = esSettings.options, processors = processors.toSeq, - aliases = aliases + aliases = aliases, + materializedViews = esMappings.materializedViews.getOrElse(Nil), + tableType = esMappings.tableType ).update() } } @@ -506,4 +592,76 @@ package object schema { ) } } + + object NamingUtils { + + /** Normalizes Elasticsearch object names (indices, pipelines, transforms, policies) + * + * Rules: + * - Only lowercase alphanumeric + underscore + dot + * - Max 255 characters + * - If too long, truncates with readable prefix + hash suffix + */ + def normalizeObjectName(name: String, maxLength: Int = 255): String = { + require(maxLength > 10, "maxLength must be > 10 to allow prefix + hash") + + // Step 1: Normalize to Elasticsearch-safe characters + // Keep: a-z, 0-9, underscore, dot + val normalized = name.toLowerCase + .replaceAll("[^a-zA-Z0-9_.]", "_") // caractères invalides -> "_" + .replaceAll("_+", "_") // compacter plusieurs "_" + .stripPrefix("_") + .stripSuffix("_") + + // Step 2: If within limit, return as-is + if (normalized.length <= maxLength) { + return normalized + } + + // Step 3: Generate readable truncated name with hash + // Format: _ + // Example: orders_with_customers_mv_orders_enri_a1b2c3d4 + + val hashLength = 8 // Short hash for readability + val prefixLength = maxLength - hashLength - 1 // -1 for underscore + + val prefix = normalized.take(prefixLength) + val hash = generateShortHash(normalized, hashLength) + + s"${prefix}_${hash}" + } + + /** Generates a short deterministic hash from a string + * + * Uses MD5 for consistency, takes first N hex chars + */ + private def generateShortHash(input: String, length: Int = 8): String = { + val digest = MessageDigest.getInstance("MD5") + val hashBytes = digest.digest(input.getBytes("UTF-8")) + hashBytes.map("%02x".format(_)).mkString.take(length) + } + + /** Validates an Elasticsearch object name + * + * @return + * Right(name) if valid, Left(error) otherwise + */ + def validateObjectName(name: String): Either[String, String] = { + if (name.isEmpty) { + Left("Name cannot be empty") + } else if (name.length > 255) { + Left(s"Name exceeds 255 characters: ${name.length}") + } else if (!name.matches("^[a-z0-9_.]+$")) { + Left( + s"Name contains invalid characters. Only lowercase alphanumeric, underscore and dot allowed: $name" + ) + } else if (name.startsWith("_") || name.startsWith("-") || name.startsWith("+")) { + Left(s"Name cannot start with _, - or +: $name") + } else if (name == "." || name == "..") { + Left("Name cannot be . or ..") + } else { + Right(name) + } + } + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala index 16bf57b1..fcf51616 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala @@ -16,6 +16,7 @@ package app.softnetwork.elastic.sql.function +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} import app.softnetwork.elastic.sql.query.{Bucket, BucketPath, Field, Limit, OrderBy, SingleSearch} import app.softnetwork.elastic.sql.{Expr, Identifier, TokenRegex, Updateable} @@ -88,7 +89,7 @@ package object aggregate { aggregations.map(_.bucketPath).distinct.sortBy(_.length).reverse.headOption.getOrElse("") override def update(request: SingleSearch): BucketScriptAggregation = { - val identifiers = FunctionUtils.aggregateIdentifiers(identifier) + val identifiers = FunctionUtils.funIdentifiers(identifier) val params = identifiers.flatMap { case identifier: Identifier => val name = identifier.metricName.getOrElse(identifier.aliasOrName) @@ -226,6 +227,10 @@ package object aggregate { partitionBy: Seq[Identifier] = Seq.empty, fields: Seq[Field] = Seq.empty ) extends WindowFunction { + override def baseType: SQLType = SQLTypes.BigInt + + def isCardinality: Boolean = identifier.distinct + override def limit: Option[Limit] = None override def orderBy: Option[OrderBy] = None diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala index 0c432619..f4df7954 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala @@ -17,12 +17,14 @@ package app.softnetwork.elastic.sql.function import app.softnetwork.elastic.sql.{ + query, Expr, Identifier, LiteralParam, PainlessContext, PainlessScript, - TokenRegex + TokenRegex, + Updateable } import app.softnetwork.elastic.sql.`type`.{ SQLAny, @@ -82,6 +84,10 @@ package object cond { s"${arg.trim} == null" // TODO check when identifier is nullable and has functions case _ => throw new IllegalArgumentException("ISNULL requires exactly one argument") } + + override def update(request: query.SingleSearch): IsNull = { + this.copy(identifier = identifier.update(request)) + } } case class IsNotNull(identifier: Identifier) extends ConditionalFunction[SQLAny] { @@ -101,6 +107,10 @@ package object cond { s"${arg.trim} != null" // TODO check when identifier is nullable and has functions case _ => throw new IllegalArgumentException("ISNOTNULL requires exactly one argument") } + + override def update(request: query.SingleSearch): IsNotNull = { + this.copy(identifier = identifier.update(request)) + } } case class Coalesce(values: List[PainlessScript]) @@ -144,6 +154,13 @@ package object cond { } override def nullable: Boolean = values.forall(_.nullable) + + override def update(request: query.SingleSearch): Coalesce = { + this.copy(values = values.map { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case other => other + }) + } } case class NullIf(expr1: PainlessScript, expr2: PainlessScript) @@ -199,6 +216,19 @@ package object cond { _ <- Validator.validateTypesMatching(expr1.out, expr2.out) } yield () } + + override def update(request: query.SingleSearch): NullIf = { + this.copy( + expr1 = expr1 match { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case other => other + }, + expr2 = expr2 match { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case other => other + } + ) + } } case class Case( @@ -211,6 +241,7 @@ package object cond { default.toList override def inputType: SQLAny = SQLTypes.Any + override def outputType: SQLAny = SQLTypes.Any override def sql: String = { @@ -362,6 +393,29 @@ package object cond { override def nullable: Boolean = conditions.exists { case (_, res) => res.nullable } || default.forall(_.nullable) - } + override def update(request: query.SingleSearch): Case = { + this.copy( + expression = expression.map { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case other => other + }, + conditions = conditions.map { case (cond, res) => + val newCond = cond match { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case other => other + } + val newRes = res match { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case other => other + } + (newCond, newRes) + }, + default = default.map { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case other => other + } + ) + } + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala index 8a71bcf4..621106f2 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala @@ -17,13 +17,15 @@ package app.softnetwork.elastic.sql.function import app.softnetwork.elastic.sql.{ + query, Alias, DateMathRounding, Expr, Identifier, PainlessContext, PainlessScript, - TokenRegex + TokenRegex, + Updateable } import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} @@ -100,6 +102,14 @@ package object convert { else s"$Cast($ret)" } value.cast(targetType) + + override def update(request: query.SingleSearch): Conversion = { + value match { + case updatable: Updateable => + this.copy(value = updatable.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case object CastOperator extends Expr("\\:\\:") with TokenRegex @@ -110,6 +120,14 @@ package object convert { override def safe: Boolean = false value.cast(targetType) + + override def update(request: query.SingleSearch): CastOperator = { + value match { + case updatable: Updateable => + this.copy(value = updatable.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case object Convert extends Expr("CONVERT") with TokenRegex @@ -120,5 +138,13 @@ package object convert { override def safe: Boolean = false value.cast(targetType) + + override def update(request: query.SingleSearch): Convert = { + value match { + case updatable: Updateable => + this.copy(value = updatable.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala index adcc4fae..44d2129b 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala @@ -17,12 +17,14 @@ package app.softnetwork.elastic.sql.function import app.softnetwork.elastic.sql.{ + query, Expr, IntValue, PainlessContext, PainlessParam, PainlessScript, - TokenRegex + TokenRegex, + Updateable } import app.softnetwork.elastic.sql.`type`.{SQLNumeric, SQLType, SQLTypes} @@ -91,12 +93,26 @@ package object math { arg: PainlessScript ) extends MathematicalFunction { override def args: List[PainlessScript] = List(arg) + override def update(request: query.SingleSearch): MathematicalFunctionWithOp = { + arg match { + case updatable: Updateable => + this.copy(arg = updatable.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case class Pow(arg: PainlessScript, exponent: Int) extends MathematicalFunction { override def mathOp: MathOp = Pow override def args: List[PainlessScript] = List(arg, IntValue(exponent)) override def nullable: Boolean = arg.nullable + override def update(request: query.SingleSearch): Pow = { + arg match { + case updatable: Updateable => + this.copy(arg = updatable.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case class PowParam(scale: Int) extends PainlessParam with PainlessScript { @@ -130,6 +146,13 @@ package object math { s"${fun.map(_.sql).getOrElse("")}($arg${scale.map(s => s", $s").getOrElse("")})" } + override def update(request: query.SingleSearch): Round = { + arg match { + case updatable: Updateable => + this.copy(arg = updatable.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case class Sign(arg: PainlessScript) extends MathematicalFunction { @@ -143,12 +166,38 @@ package object math { case _ => throw new IllegalArgumentException("Sign function requires exactly one argument") } + override def update(request: query.SingleSearch): Sign = { + arg match { + case updatable: Updateable => + this.copy(arg = updatable.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case class Atan2(y: PainlessScript, x: PainlessScript) extends MathematicalFunction { override def mathOp: MathOp = Atan2 override def args: List[PainlessScript] = List(y, x) override def nullable: Boolean = y.nullable || x.nullable + + override def update(request: query.SingleSearch): Atan2 = { + (y, x) match { + case (updatableY: Updateable, updatableX: Updateable) => + this.copy( + y = updatableY.update(request).asInstanceOf[PainlessScript], + x = updatableX.update(request).asInstanceOf[PainlessScript] + ) + case (updatableY: Updateable, _) => + this.copy( + y = updatableY.update(request).asInstanceOf[PainlessScript] + ) + case (_, updatableX: Updateable) => + this.copy( + x = updatableX.update(request).asInstanceOf[PainlessScript] + ) + case _ => this + } + } } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index 064e6b25..67fb5d4c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -18,6 +18,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.function.aggregate.AggregateFunction +import app.softnetwork.elastic.sql.function.time.CurrentFunction import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression import app.softnetwork.elastic.sql.parser.Validator import app.softnetwork.elastic.sql.query.{NestedElement, SingleSearch} @@ -34,9 +35,14 @@ package object function { def expr: Token = _expr override def nullable: Boolean = expr.nullable def functionNestedElement: Option[NestedElement] = None + def dependencies: Seq[Identifier] = FunctionUtils.funIdentifiers(this).filterNot(_.name.isEmpty) + def usesCurrentTimeFunction: Boolean = this match { + case _: CurrentFunction => true + case _ => false + } } - trait FunctionWithIdentifier extends Function { + trait FunctionWithIdentifier extends Function with Updateable { def identifier: Identifier override def functionNestedElement: Option[NestedElement] = @@ -91,31 +97,29 @@ package object function { } } - def aggregateIdentifiers( + def funIdentifiers( fun: Function, - acc: Seq[FunctionChain] = Seq.empty - ): Seq[FunctionChain] = { + acc: Seq[Identifier] = Seq.empty + ): Seq[Identifier] = { fun match { - case fwi: FunctionWithIdentifier => aggregateIdentifiers(fwi.identifier, acc) - case fc: FunctionChain => - fc.functions.foldLeft(acc) { - case (innerAcc, _: AggregateFunction) => innerAcc :+ fc - case (innerAcc, i: FunctionWithIdentifier) => - aggregateIdentifiers(i.identifier, innerAcc) - case (innerAcc, fc: FunctionChain) => aggregateIdentifiers(fc, innerAcc) - case (innerAcc, b: BinaryFunction[_, _, _]) => aggregateIdentifiers(b, innerAcc) - case (innerAcc, _) => innerAcc + case id: Identifier => + id.functions.foldLeft(acc :+ id) { case (innerAcc, fun: Function) => + funIdentifiers(fun, innerAcc) } - case b: BinaryFunction[_, _, _] => - val leftAcc = b.left match { - case f: Function => aggregateIdentifiers(f, acc) - case _ => acc - } - b.right match { - case f: Function => aggregateIdentifiers(f, leftAcc) - case _ => leftAcc + case fn: FunctionN[_, _] => + fn.args + .collect { case f: Function => + f + } + .foldLeft(acc) { (innerAcc, f) => + funIdentifiers(f, innerAcc) + } + case fc: FunctionChain => + fc.functions.foldLeft(acc) { case (innerAcc, fun: Function) => + funIdentifiers(fun, innerAcc) } - case _ => acc + case fwi: FunctionWithIdentifier => funIdentifiers(fwi.identifier, acc :+ fwi.identifier) + case _ => acc } } } @@ -205,9 +209,22 @@ package object function { override def functionNestedElement: Option[NestedElement] = functions.flatMap(_.functionNestedElement).headOption + + override def usesCurrentTimeFunction: Boolean = { + functions.exists { _.usesCurrentTimeFunction } + } + + override lazy val dependencies: Seq[Identifier] = functions + .foldLeft(Seq.empty[Identifier]) { case (acc, fun) => + acc ++ FunctionUtils.funIdentifiers(fun) + } + .filterNot(_.name.isEmpty) } - trait FunctionN[In <: SQLType, Out <: SQLType] extends Function with PainlessScript { + trait FunctionN[In <: SQLType, Out <: SQLType] + extends Function + with PainlessScript + with Updateable { def fun: Option[PainlessScript] = None def args: List[PainlessScript] @@ -350,6 +367,11 @@ package object function { s"${fun.map(_.painless(context)).getOrElse("")}(${callArgs.mkString(argsSeparator)})" else fun.map(_.painless(context)).getOrElse("") + + override def usesCurrentTimeFunction: Boolean = this.args.exists { + case f: Function => f.usesCurrentTimeFunction + case _ => false + } } trait BinaryFunction[In1 <: SQLType, In2 <: SQLType, Out <: SQLType] extends FunctionN[In2, Out] { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala index bbd98d01..6c3529cf 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala @@ -16,7 +16,15 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.{Expr, IntValue, PainlessContext, PainlessScript, TokenRegex} +import app.softnetwork.elastic.sql.{ + query, + Expr, + IntValue, + PainlessContext, + PainlessScript, + TokenRegex, + Updateable +} import app.softnetwork.elastic.sql.`type`.{ SQLBigInt, SQLBool, @@ -133,6 +141,12 @@ package object string { extends StringFunction[SQLVarchar] { override def outputType: SQLVarchar = SQLTypes.Varchar override def args: List[PainlessScript] = List(str) + override def update(request: query.SingleSearch): StringFunctionWithOp = { + str match { + case u: Updateable => this.copy(str = u.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case class Substring(str: PainlessScript, start: Int, length: Option[Int]) @@ -172,6 +186,12 @@ package object string { override def toSQL(base: String): String = sql + override def update(request: query.SingleSearch): Substring = { + str match { + case u: Updateable => this.copy(str = u.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case class Concat(values: List[PainlessScript]) extends StringFunction[SQLVarchar] { @@ -182,6 +202,16 @@ package object string { override def nullable: Boolean = values.exists(_.nullable) + override def update(request: query.SingleSearch): Concat = { + values.map { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case v => v + } match { + case updatedValues => this.copy(values = updatedValues) + case _ => this + } + } + override def toPainlessCall( callArgs: List[String], context: Option[PainlessContext] @@ -217,6 +247,12 @@ package object string { override def outputType: SQLBigInt = SQLTypes.BigInt override def stringOp: StringOp = Length override def args: List[PainlessScript] = List(str) + override def update(request: query.SingleSearch): Length = { + str match { + case u: Updateable => this.copy(str = u.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case class LeftFunction(str: PainlessScript, length: Int) extends StringFunction[SQLVarchar] { @@ -245,6 +281,13 @@ package object string { str.validate() override def toSQL(base: String): String = sql + + override def update(request: query.SingleSearch): LeftFunction = { + str match { + case u: Updateable => this.copy(str = u.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case class RightFunction(str: PainlessScript, length: Int) extends StringFunction[SQLVarchar] { @@ -275,6 +318,13 @@ package object string { str.validate() override def toSQL(base: String): String = sql + + override def update(request: query.SingleSearch): RightFunction = { + str match { + case u: Updateable => this.copy(str = u.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case class Replace(str: PainlessScript, search: PainlessScript, replace: PainlessScript) @@ -304,6 +354,21 @@ package object string { } override def toSQL(base: String): String = sql + + override def update(request: query.SingleSearch): Replace = { + List(str, search, replace).map { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case v => v + } match { + case List(updatedStr, updatedSearch, updatedReplace) => + this.copy( + str = updatedStr, + search = updatedSearch, + replace = updatedReplace + ) + case _ => this + } + } } case class Reverse(str: PainlessScript) extends StringFunction[SQLVarchar] { @@ -328,6 +393,13 @@ package object string { str.validate() override def toSQL(base: String): String = sql + + override def update(request: query.SingleSearch): Reverse = { + str match { + case u: Updateable => this.copy(str = u.update(request).asInstanceOf[PainlessScript]) + case _ => this + } + } } case class Position(substr: PainlessScript, str: PainlessScript, start: Int) @@ -360,6 +432,20 @@ package object string { } override def toSQL(base: String): String = sql + + override def update(request: query.SingleSearch): Position = { + List(substr, str).map { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case v => v + } match { + case List(updatedSubstr, updatedStr) => + this.copy( + substr = updatedSubstr, + str = updatedStr + ) + case _ => this + } + } } case class RegexpLike( @@ -394,5 +480,20 @@ package object string { } override def toSQL(base: String): String = sql + + override def update(request: query.SingleSearch): RegexpLike = { + List(str, pattern).map { + case u: Updateable => u.update(request).asInstanceOf[PainlessScript] + case v => v + } match { + case List(updatedStr, updatedPattern) => + this.copy( + str = updatedStr, + pattern = updatedPattern, + matchFlags = matchFlags + ) + case _ => this + } + } } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index 1856002f..a04b77e5 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -17,6 +17,7 @@ package app.softnetwork.elastic.sql.function import app.softnetwork.elastic.sql.{ + query, DateMathRounding, DateMathScript, Expr, @@ -25,7 +26,8 @@ import app.softnetwork.elastic.sql.{ PainlessContext, PainlessScript, StringValue, - TokenRegex + TokenRegex, + Updateable } import app.softnetwork.elastic.sql.operator.time._ import app.softnetwork.elastic.sql.`type`.{ @@ -109,11 +111,13 @@ package object time { case class SQLAddInterval(interval: TimeInterval) extends AddInterval[SQLTemporal] { override def inputType: SQLTemporal = SQLTypes.Temporal override def outputType: SQLTemporal = SQLTypes.Temporal + override def update(request: query.SingleSearch): SQLAddInterval = this } case class SQLSubtractInterval(interval: TimeInterval) extends SubtractInterval[SQLTemporal] { override def inputType: SQLTemporal = SQLTypes.Temporal override def outputType: SQLTemporal = SQLTypes.Temporal + override def update(request: query.SingleSearch): SQLSubtractInterval = this } sealed trait DateTimeFunction extends Function { @@ -283,6 +287,9 @@ package object time { override def shouldBeScripted: Boolean = false + override def update(request: query.SingleSearch): DateTrunc = { + this.copy(identifier = identifier.update(request)) + } } case object Extract extends Expr("EXTRACT") with TokenRegex with PainlessScript { @@ -304,6 +311,7 @@ package object time { override def toSQL(base: String): String = s"$sql(${field.sql} FROM $base)" + override def update(request: query.SingleSearch): Extract = this } import TimeField._ @@ -393,6 +401,9 @@ package object time { } } + override def update(request: query.SingleSearch): LastDayOfMonth = { + this.copy(identifier = identifier.update(request)) + } } case object DateDiff extends Expr("DATE_DIFF") with TokenRegex with PainlessScript { @@ -435,6 +446,21 @@ package object time { } ret } + + override def update(request: query.SingleSearch): DateDiff = { + this.copy( + start = start match { + case updatable: Updateable => + updatable.update(request).asInstanceOf[PainlessScript] + case _ => start + }, + end = end match { + case updatable: Updateable => + updatable.update(request).asInstanceOf[PainlessScript] + case _ => end + } + ) + } } case object DateAdd extends Expr("DATE_ADD") with TokenRegex { @@ -453,6 +479,9 @@ package object time { s"$sql($base, ${interval.sql})" } override def dateMathScript: Boolean = identifier.dateMathScript + override def update(request: query.SingleSearch): DateAdd = { + this.copy(identifier = identifier.update(request)) + } } case object DateSub extends Expr("DATE_SUB") with TokenRegex { @@ -471,6 +500,9 @@ package object time { s"$sql($base, ${interval.sql})" } override def dateMathScript: Boolean = identifier.dateMathScript + override def update(request: query.SingleSearch): DateSub = { + this.copy(identifier = identifier.update(request)) + } } sealed trait FunctionWithDateTimeFormat { @@ -579,6 +611,10 @@ package object time { override def formatScript: Option[String] = Some(format) override def shouldBeScripted: Boolean = true // FIXME + + override def update(request: query.SingleSearch): DateParse = { + this.copy(identifier = identifier.update(request)) + } } case object DateFormat extends Expr("DATE_FORMAT") with TokenRegex with PainlessScript { @@ -625,6 +661,10 @@ package object time { s"$param.format($arg)" case _ => throw new IllegalArgumentException("DateParse requires exactly one argument") } + + override def update(request: query.SingleSearch): DateFormat = { + this.copy(identifier = identifier.update(request)) + } } case object DateTimeAdd extends Expr("DATETIME_ADD") with TokenRegex { @@ -643,6 +683,10 @@ package object time { s"$sql($base, ${interval.sql})" } override def dateMathScript: Boolean = identifier.dateMathScript + + override def update(request: query.SingleSearch): DateTimeAdd = { + this.copy(identifier = identifier.update(request)) + } } case object DateTimeSub extends Expr("DATETIME_SUB") with TokenRegex { @@ -661,6 +705,10 @@ package object time { s"$sql($base, ${interval.sql})" } override def dateMathScript: Boolean = identifier.dateMathScript + + override def update(request: query.SingleSearch): DateTimeSub = { + this.copy(identifier = identifier.update(request)) + } } case object DateTimeParse extends Expr("DATETIME_PARSE") with TokenRegex with PainlessScript { @@ -720,6 +768,10 @@ package object time { } override def formatScript: Option[String] = Some(format) + + override def update(request: query.SingleSearch): DateTimeParse = { + this.copy(identifier = identifier.update(request)) + } } case object DateTimeFormat extends Expr("DATETIME_FORMAT") with TokenRegex with PainlessScript { @@ -769,6 +821,9 @@ package object time { case _ => throw new IllegalArgumentException("DateParse requires exactly one argument") } + override def update(request: query.SingleSearch): DateTimeFormat = { + this.copy(identifier = identifier.update(request)) + } } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index ad54ce26..ded20f04 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -16,6 +16,7 @@ package app.softnetwork.elastic +import app.softnetwork.elastic.schema.NamingUtils import app.softnetwork.elastic.sql.function.aggregate.{AggregateFunction, COUNT, WindowFunction} import app.softnetwork.elastic.sql.function.geo.DistanceUnit import app.softnetwork.elastic.sql.function.time.CurrentFunction @@ -24,7 +25,6 @@ import app.softnetwork.elastic.sql.query._ import app.softnetwork.elastic.sql.schema.Column import com.fasterxml.jackson.databind.JsonNode -import java.security.MessageDigest import scala.annotation.tailrec import scala.jdk.CollectionConverters._ import scala.reflect.runtime.universe._ @@ -136,6 +136,7 @@ package object sql { case object PainlessContextType { case object Processor extends PainlessContextType case object Query extends PainlessContextType + case object Transform extends PainlessContextType } /** Context for painless scripts @@ -154,10 +155,13 @@ package object sql { def isProcessor: Boolean = context == PainlessContextType.Processor + def isTransform: Boolean = context == PainlessContextType.Transform + lazy val timestamp: String = { context match { case PainlessContextType.Processor => CurrentFunction.processorTimestamp - case PainlessContextType.Query => CurrentFunction.queryTimestamp + case PainlessContextType.Query | PainlessContextType.Transform => + CurrentFunction.queryTimestamp } } @@ -171,9 +175,19 @@ package object sql { def addParam(token: Token): Option[String] = { token match { case identifier: Identifier if isProcessor => - addParam( - LiteralParam(identifier.processParamName, None /*identifier.processCheckNotNull*/ ) - ) + if (identifier.name.nonEmpty) + addParam( + LiteralParam(identifier.processParamName, None /*identifier.processCheckNotNull*/ ) + ) + else + None + case identifier: Identifier if isTransform => + if (identifier.name.nonEmpty) + addParam( + LiteralParam(identifier.transformParamName, identifier.transformCheckNotNull) + ) + else + None case param: PainlessParam if param.param.nonEmpty && (param.isInstanceOf[LiteralParam] || param.nullable) => get(param) match { @@ -198,6 +212,10 @@ package object sql { token match { case identifier: Identifier if isProcessor => get(LiteralParam(identifier.processParamName, None /*identifier.processCheckNotNull*/ )) + case identifier: Identifier if isTransform => + get( + LiteralParam(identifier.transformParamName, None /*identifier.transformCheckNotNull*/ ) + ) case param: PainlessParam => if (exists(param)) Try(_values(_keys.indexOf(param))).toOption else None @@ -323,7 +341,7 @@ package object sql { case f: Float => FloatValue(f) case d: Double => DoubleValue(d) case a: Array[T] => apply(a.toSeq) - case a: Seq[T] => + case a: Seq[T] if a.nonEmpty => val values = a.map(apply) values.headOption match { case Some(_: StringValue) => @@ -350,6 +368,7 @@ package object sql { ).asInstanceOf[Values[R, T]] case _ => throw new IllegalArgumentException("Unsupported Values type") } + case _: Seq[T] => EmptyValues().asInstanceOf[Values[R, T]] case o: Map[_, _] => val map = o.asInstanceOf[Map[String, Any]].map { case (k, v) => k -> apply(v) } ObjectValue(map) @@ -768,6 +787,11 @@ package object sql { def toJson: JsonNode = this } + case class EmptyValues() extends Values[Any, Null](Seq.empty) { + override def sql: String = "()" + override def nullable: Boolean = true + } + def toRegex(value: String): String = { value.replaceAll("%", ".*").replaceAll("_", ".") } @@ -793,21 +817,7 @@ package object sql { acc.replace(k, s"_${v}_") } // Nettoyer pour obtenir un identifiant valide - val normalized = replaced - .replaceAll("[^a-zA-Z0-9_]", "_") // caractères invalides -> "_" - .replaceAll("_+", "_") // compacter plusieurs "_" - .stripPrefix("_") - .stripSuffix("_") - .toLowerCase - - // Tronquer si nécessaire - if (normalized.length > MaxAliasLength) { - val digest = MessageDigest.getInstance("MD5").digest(normalized.getBytes("UTF-8")) - val hash = digest.map("%02x".format(_)).mkString.take(8) // suffix court - normalized.take(MaxAliasLength - hash.length - 1) + "_" + hash - } else { - normalized - } + NamingUtils.normalizeObjectName(replaced, MaxAliasLength).replaceAll("\\.", "_") } } @@ -990,7 +1000,23 @@ package object sql { lazy val processCheckNotNull: Option[String] = if (path.isEmpty || !nullable) None else - Option(s"(ctx.$path == null ? $nullValue : ctx.$path${painlessMethods.mkString("")})") + Option( + s"($processParamName == null ? $nullValue : $processParamName${painlessMethods.mkString("")})" + ) + + lazy val transformParamName: String = + if (isAggregation && functions.size == 1) s"params.${metricName.getOrElse(aliasOrName)}" + else if (aliasOrName.nonEmpty) + s"doc['$aliasOrName'].value" + else "" + + lazy val transformCheckNotNull: Option[String] = + if (aliasOrName.isEmpty || !nullable) None + else + Option( + s"(doc['$aliasOrName'].size() == 0 ? $nullValue : doc['$aliasOrName'].value${painlessMethods + .mkString("")})" + ) def originalType: SQLType = if (name.trim.nonEmpty) SQLTypes.Any @@ -1074,6 +1100,14 @@ package object sql { def isWindowing: Boolean = windows.exists(_.partitionBy.nonEmpty) + def painlessScriptRequired: Boolean = functions.nonEmpty && !hasAggregation && bucket.isEmpty + + def isObject: Boolean = { + out match { + case SQLTypes.Struct => true + case _ => name.contains(".") + } + } } object Identifier { @@ -1126,7 +1160,7 @@ package object sql { } val parts: Seq[String] = name.split("\\.").toSeq val tableAlias = parts.head - val table = request.tableAliases.find(t => t._2 == tableAlias).map(_._2) + val table = request.tableAliases.find(t => t._2 == tableAlias).map(_._1) if (table.nonEmpty) { request.unnestAliases.find(_._1 == tableAlias) match { case Some(tuple) if !nested => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/FromParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/FromParser.scala index b18ea945..6d4fd964 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/FromParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/FromParser.scala @@ -16,6 +16,7 @@ package app.softnetwork.elastic.sql.parser +import app.softnetwork.elastic.sql.{Alias, Identifier} import app.softnetwork.elastic.sql.query.{ CrossJoin, From, @@ -26,6 +27,7 @@ import app.softnetwork.elastic.sql.query.{ LeftJoin, On, RightJoin, + StandardJoin, Table, Unnest } @@ -33,7 +35,7 @@ import app.softnetwork.elastic.sql.query.{ trait FromParser { self: Parser with WhereParser with LimitParser => - def unnest: PackratParser[Unnest] = + def unnest: PackratParser[Join] = Unnest.regex ~ start ~ identifier ~ end ~ alias.? ^^ { case _ ~ i ~ _ ~ a => Unnest(i, None, a) } @@ -52,8 +54,23 @@ trait FromParser { ) } - def join: PackratParser[Join] = opt(join_type) ~ Join.regex ~ unnest ~ opt(on) ^^ { - case jt ~ _ ~ t ~ o => t // Unnest cannot have a join type or an ON clause + def source: PackratParser[(Identifier, Option[Alias])] = identifier ~ alias.? ^^ { case i ~ a => + (i, a) + } + + def join: PackratParser[Join] = opt(join_type) ~ Join.regex ~ (unnest | source) ~ opt(on) ^^ { + case jt ~ _ ~ t ~ o => + t match { + case u: Unnest => + u // Unnest cannot have a join type or an ON clause + case (i: Identifier, a: Option[Alias]) => + StandardJoin( + source = i, + joinType = jt, + on = o, + alias = a + ) + } } def table: PackratParser[Table] = identifierRegex ~ alias.? ~ rep(join) ^^ { case i ~ a ~ js => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 81ddacf5..e5f951fc 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -32,11 +32,13 @@ import app.softnetwork.elastic.sql.parser.operator.math.ArithmeticParser import app.softnetwork.elastic.sql.query._ import app.softnetwork.elastic.sql.schema.{ Column, + Frequency, IngestPipelineType, IngestProcessor, IngestProcessorType, PartitionDate, - ScriptProcessor + ScriptProcessor, + TransformTimeUnit } import app.softnetwork.elastic.sql.time.TimeUnit @@ -133,6 +135,7 @@ object Parser case "rename" => IngestProcessorType.Rename case "remove" => IngestProcessorType.Remove case "date_index_name" => IngestProcessorType.DateIndexName + case "enrich" => IngestProcessorType.Enrich case other => IngestProcessorType(other) } } @@ -174,7 +177,7 @@ object Parser } def describePipeline: PackratParser[DescribePipeline] = - ("DESCRIBE" ~ "PIPELINE") ~ ident ^^ { case _ ~ pipeline => + (("DESCRIBE" | "DESC") ~ "PIPELINE") ~ ident ^^ { case _ ~ pipeline => DescribePipeline(pipeline) } @@ -363,6 +366,11 @@ object Parser } } + def showTables: PackratParser[ShowTables.type] = + ("SHOW" ~ "TABLES") ^^ { _ => + ShowTables + } + def showTable: PackratParser[ShowTable] = ("SHOW" ~ "TABLE") ~ ident ^^ { case _ ~ table => ShowTable(table) @@ -374,7 +382,7 @@ object Parser } def describeTable: PackratParser[DescribeTable] = - ("DESCRIBE" ~ "TABLE") ~ ident ^^ { case _ ~ table => + (("DESCRIBE" | "DESC") ~ "TABLE") ~ ident ^^ { case _ ~ table => DescribeTable(table) } @@ -388,6 +396,83 @@ object Parser TruncateTable(name) } + def frequency: PackratParser[Frequency] = + ("REFRESH" ~ "EVERY") ~> """\d+\s+(MILLISECOND|SECOND|MINUTE|HOUR|DAY|WEEK|MONTH|YEAR)S?""".r ^^ { + str => + val parts = str.trim.split("\\s+") + Frequency(TransformTimeUnit(parts(1)), parts(0).toInt) + } + + def withOptions: PackratParser[Map[String, Value[_]]] = + ("WITH" ~ lparen) ~> repsep(option, separator) <~ rparen ^^ { opts => + opts.toMap + } + + def createOrReplaceMaterializedView: PackratParser[CreateMaterializedView] = + ("CREATE" ~ "OR" ~ "REPLACE" ~ "MATERIALIZED" ~ "VIEW") ~ ident ~ opt(frequency) ~ opt( + withOptions + ) ~ ("AS" ~> dqlStatement) ^^ { case _ ~ view ~ freq ~ opts ~ dql => + CreateMaterializedView( + view, + dql, + ifNotExists = false, + orReplace = true, + frequency = freq, + options = opts.getOrElse(Map.empty) + ) + } + + def createMaterializedView: PackratParser[CreateMaterializedView] = + ("CREATE" ~ "MATERIALIZED" ~ "VIEW") ~ ifNotExists ~ ident ~ opt( + frequency + ) ~ opt( + withOptions + ) ~ ("AS" ~> dqlStatement) ^^ { case _ ~ ine ~ view ~ freq ~ opts ~ dql => + CreateMaterializedView( + view, + dql, + ifNotExists = ine, + orReplace = false, + frequency = freq, + options = opts.getOrElse(Map.empty) + ) + } + + def dropMaterializedView: PackratParser[DropMaterializedView] = + ("DROP" ~ "MATERIALIZED" ~ "VIEW") ~ ifExists ~ ident ^^ { case _ ~ ie ~ name => + DropMaterializedView(name, ifExists = ie) + } + + def refreshMaterializedView: PackratParser[RefreshMaterializedView] = + ("REFRESH" ~ "MATERIALIZED" ~ "VIEW") ~ ident ^^ { case _ ~ _ ~ view => + RefreshMaterializedView(view) + } + + def showMaterializedViewStatus: PackratParser[ShowMaterializedViewStatus] = + ("SHOW" ~ "MATERIALIZED" ~ "VIEW" ~ "STATUS") ~ ident ^^ { case _ ~ _ ~ _ ~ _ ~ view => + ShowMaterializedViewStatus(view) + } + + def showCreateMaterializedView: PackratParser[ShowCreateMaterializedView] = + ("SHOW" ~ "CREATE" ~ "MATERIALIZED" ~ "VIEW") ~ ident ^^ { case _ ~ _ ~ _ ~ _ ~ view => + ShowCreateMaterializedView(view) + } + + def showMaterializedView: PackratParser[ShowMaterializedView] = + ("SHOW" ~ "MATERIALIZED" ~ "VIEW") ~ ident ^^ { case _ ~ _ ~ view => + ShowMaterializedView(view) + } + + def showMaterializedViews: PackratParser[ShowMaterializedViews.type] = + ("SHOW" ~ "MATERIALIZED" ~ "VIEWS") ^^ { _ => + ShowMaterializedViews + } + + def describeMaterializedView: PackratParser[DescribeMaterializedView] = + (("DESCRIBE" | "DESC") ~ "MATERIALIZED" ~ "VIEW") ~ ident ^^ { case _ ~ _ ~ _ ~ view => + DescribeMaterializedView(view) + } + def addColumn: PackratParser[AddColumn] = ("ADD" ~ "COLUMN") ~ ifNotExists ~ column ^^ { case _ ~ ine ~ col => AddColumn(col, ifNotExists = ine) @@ -581,13 +666,23 @@ object Parser alterPipeline | dropTable | truncateTable | + showTables | showTable | showCreateTable | describeTable | dropPipeline | showPipeline | showCreatePipeline | - describePipeline + describePipeline | + createMaterializedView | + createOrReplaceMaterializedView | + dropMaterializedView | + refreshMaterializedView | + showMaterializedViewStatus | + showMaterializedViews | + showMaterializedView | + showCreateMaterializedView | + describeMaterializedView def onConflict: PackratParser[OnConflict] = ("ON" ~ "CONFLICT" ~> opt(conflictTarget) <~ "DO") ~ ("UPDATE" | "NOTHING") ^^ { @@ -896,7 +991,7 @@ trait Parser }) >> cast private val regexAlias = - s"""\\b(?i)(?!(?:${reservedKeywords.mkString("|")})\\b)[a-zA-Z0-9_]*""".stripMargin + s"""\\b(?i)(?!(?:${reservedKeywords.mkString("|")})\\b)[a-zA-Z0-9_.]*""".stripMargin def alias: PackratParser[Alias] = Alias.regex.? ~ regexAlias.r ^^ { case _ ~ b => Alias(b) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala index 3ad17781..38962d76 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala @@ -16,12 +16,14 @@ package app.softnetwork.elastic.sql.query +import app.softnetwork.elastic.sql.operator.{AND, EQ} import app.softnetwork.elastic.sql.{ asString, Alias, Expr, Identifier, Source, + Token, TokenRegex, Updateable } @@ -44,9 +46,99 @@ case object CrossJoin extends Expr("CROSS") with JoinType case object On extends Expr("ON") with TokenRegex +case class JoinKey(table: String, tableAlias: String, field: String) extends Token { + def sql: String = s"$tableAlias.$field" + lazy val key: String = s"${table}_$field" +} + +object JoinKey { + def apply(identifier: Identifier): Option[JoinKey] = { + identifier.table match { + case Some(ta) if identifier.name.nonEmpty => + Some(JoinKey(ta, identifier.tableAlias.getOrElse(ta), identifier.name)) + case _ => None + } + } +} + +case class JoinKeyMatch(left: JoinKey, right: JoinKey) extends Token { + def sql: String = s"${left.sql} = ${right.sql}" + + lazy val joinKeys: Seq[JoinKey] = Seq(left, right) +} + +object JoinKeyMatch { + def apply(expression: Expression): Option[JoinKeyMatch] = { + if (expression.operator != EQ) return None + (expression.identifier, expression.maybeValue) match { + case (leftId: Identifier, Some(rightId: Identifier)) => + (JoinKey(leftId), JoinKey(rightId)) match { + case (Some(leftKey), Some(rightKey)) => + Some(JoinKeyMatch(leftKey, rightKey)) + case _ => None + } + case _ => None + } + } +} + case class On(criteria: Criteria) extends Updateable { override def sql: String = s" $On $criteria" def update(request: SingleSearch): On = this.copy(criteria = criteria.update(request)) + + private def extractJoinKeyMatches(criteria: Criteria): Seq[JoinKeyMatch] = { + criteria match { + case e: Expression if e.operator == EQ => JoinKeyMatch(e).toSeq + case p: Predicate if p.operator == AND => + extractJoinKeyMatches(p.leftCriteria) ++ extractJoinKeyMatches(p.rightCriteria) + case _ => Seq.empty + } + } + + /* Extracted join keys from the ON criteria */ + lazy val joinKeyMatches: Seq[JoinKeyMatch] = extractJoinKeyMatches(criteria) + + private def validateOnCriteria(criteria: Criteria): Either[String, Unit] = { + criteria match { + case e: Expression if e.operator == EQ => + if (e.identifier.name.isEmpty) Left(s"ON clause $this identifier cannot be empty") + else if (e.identifier.functions.nonEmpty) + Left(s"ON clause $this cannot use functions in equality expressions") + else if (e.maybeNot.isDefined) + Left(s"ON clause $this cannot use NOT operator in equality expressions") + else if (e.maybeValue.isEmpty) + Left(s"ON clause $this equality expressions must compare two identifiers") + else { + e.maybeValue.get match { + case id: Identifier => + if (id.name.isEmpty) + Left(s"ON clause $this identifier cannot be empty") + else if (id.functions.nonEmpty) + Left(s"ON clause $this cannot use functions in equality expressions") + else Right(()) + case _ => Left(s"ON clause $this equality expressions must compare two identifiers") + } + } + case p: Predicate => + p.operator match { + case AND => + for { + _ <- validateOnCriteria(p.leftCriteria) + _ <- validateOnCriteria(p.rightCriteria) + } yield () + case _ => Left(s"ON clause $this must use AND predicate operator") + } + case _ => Left(s"ON clause $this must use either equality operator or AND predicate") + } + } + + override def validate(): Either[String, Unit] = { + for { + _ <- super.validate() + _ <- criteria.validate() + _ <- validateOnCriteria(criteria) + } yield () + } } case object Join extends Expr("JOIN") with TokenRegex @@ -57,7 +149,7 @@ sealed trait Join extends Updateable { def on: Option[On] def alias: Option[Alias] override def sql: String = - s" ${asString(joinType)} $Join $source${asString(on)}${asString(alias)}" + s" ${asString(joinType)} $Join $source${asString(alias)}${asString(on)}" override def update(request: SingleSearch): Join @@ -136,6 +228,36 @@ case class Unnest( } +case class StandardJoin( + source: Source, + joinType: Option[JoinType], // INNER JOIN by default + on: Option[On], + alias: Option[Alias] = None +) extends Join { + override def update(request: SingleSearch): StandardJoin = { + val updated = this.copy( + source = source.update(request), + on = on.map(_.update(request)) + ) + updated + } + + override def validate(): Either[String, Unit] = { + for { + _ <- joinType match { + case Some(InnerJoin | LeftJoin) => Right(()) + case None => Right(()) // by default INNER JOIN + case _ => Left(s"Standard JOIN $this requires an INNER (default) or LEFT JOIN type") + } + _ <- on match { + case Some(o) => o.validate() + case None => Left(s"Standard JOIN $this requires an ON clause") + } + _ <- super.validate() + } yield () + } +} + case class Table(name: String, tableAlias: Option[Alias] = None, joins: Seq[Join] = Nil) extends Source { override def sql: String = s"$name${asString(tableAlias)} ${joins.map(_.sql).mkString(" ")}".trim @@ -153,16 +275,18 @@ case class Table(name: String, tableAlias: Option[Alias] = None, joins: Seq[Join case errors => Left(errors.map { case Left(err) => err }.mkString("\n")) } } yield () + + lazy val joinedTables: Seq[String] = joins.collect { case sj: StandardJoin => + sj.source.name + } + + lazy val enrichmentRequired: Boolean = joinedTables.nonEmpty + } case class From(tables: Seq[Table]) extends Updateable { override def sql: String = s" $From ${tables.map(_.sql).mkString(",")}" - lazy val unnests: Seq[Unnest] = tables - .map(_.joins) - .collect { case j => - j.collect { case u: Unnest => u } - } - .flatten + lazy val unnests: Seq[Unnest] = joins.collect { case u: Unnest => u } lazy val tableAliases: Map[String, String] = tables .flatMap((table: Table) => @@ -171,13 +295,31 @@ case class From(tables: Seq[Table]) extends Updateable { case _ => Some(table.name -> table.name) } ) - .toMap ++ unnestAliases.map(unnest => unnest._2._1 -> unnest._1) + .toMap ++ unnestAliases.map(unnest => unnest._2._1 -> unnest._1) ++ joinAliases.map(join => + join._2._1 -> join._1 + ) + + lazy val aliasesToTable: Map[String, String] = tableAliases.map(_.swap) + + lazy val joins: Seq[Join] = tables.flatMap(_.joins) + + lazy val joinAliases: Map[String, (String, Option[On])] = joins.collect { case sj: StandardJoin => + ( + sj.alias + .map(_.alias) + .getOrElse( + sj.source.name + ), + (sj.source.name, sj.on) + ) + }.toMap lazy val unnestAliases: Map[String, (String, Option[Limit])] = unnests .map(u => // extract unnest info (u.alias.map(_.alias).getOrElse(u.name), (u.name, u.limit)) ) .toMap + def update(request: SingleSearch): From = this.copy(tables = tables.map(_.update(request))) @@ -197,6 +339,10 @@ case class From(tables: Seq[Table]) extends Updateable { } lazy val mainTable: Table = tables.head + + lazy val joinedTables: Seq[String] = tables.flatMap(_.joinedTables) + + lazy val enrichmentRequired: Boolean = joinedTables.nonEmpty } case class NestedElement( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala index 4437309d..16ded040 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala @@ -117,6 +117,8 @@ case class Bucket( */ override def painless(context: Option[PainlessContext]): String = identifier.painless(context) + + def isObject: Boolean = identifier.isObject } case class BucketPath(buckets: Seq[Bucket]) { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala index 243116f7..9ccd1b42 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala @@ -32,4 +32,14 @@ case class Having(criteria: Option[Criteria]) extends Updateable { def nestedElements: Seq[NestedElement] = criteria.map(_.nestedElements).getOrElse(Seq.empty).groupBy(_.path).map(_._2.head).toList + + def script: Option[String] = criteria.flatMap { criteria => + val fullScript = MetricSelectorScript + .metricSelector(criteria) + .replaceAll("1 == 1 &&", "") + .replaceAll("&& 1 == 1", "") + .replaceAll("1 == 1", "") + .trim + if (fullScript.nonEmpty) Some(fullScript) else None + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala index 45d47194..e826241f 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala @@ -47,9 +47,11 @@ case class Field( with DateMathScript { def tableAlias: Option[String] = identifier.tableAlias def table: Option[String] = identifier.table - def isScriptField: Boolean = - functions.nonEmpty && !hasAggregation && identifier.bucket.isEmpty + def isScriptField: Boolean = identifier.painlessScriptRequired override def sql: String = s"$identifier${asString(fieldAlias)}" + + override lazy val dependencies: Seq[Identifier] = identifier.dependencies + lazy val sourceField: String = { if (identifier.nested) { tableAlias @@ -107,6 +109,8 @@ case class Field( lazy val path: String = identifier.path def isBucketScript: Boolean = !isAggregation && hasAggregation + + def isObject: Boolean = identifier.isObject } case object Except extends Expr("except") with TokenRegex @@ -126,6 +130,7 @@ case class Select( lazy val fieldAliases: Map[String, String] = fields.flatMap { field => field.fieldAlias.map(a => field.identifier.identifierName -> a.alias) }.toMap + lazy val aliasesToMap: Map[String, String] = fieldAliases.map(_.swap) def update(request: SingleSearch): Select = this.copy(fields = fields.map(_.update(request)), except = except.map(_.update(request))) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index 58cffbfd..ddd4acc4 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -31,11 +31,11 @@ case object Where extends Expr("WHERE") with TokenRegex sealed trait Criteria extends Updateable with PainlessScript { def operator: Operator - def identifiers: Seq[Identifier] = this match { - case Predicate(left, _, right, _, _) => left.identifiers ++ right.identifiers - case c: Expression => c.identifiers - case relation: ElasticRelation => relation.criteria.identifiers - case m: MultiMatchCriteria => m.identifiers + def dependencies: Seq[Identifier] = this match { + case Predicate(left, _, right, _, _) => left.dependencies ++ right.dependencies + case c: Expression => c.dependencies + case relation: ElasticRelation => relation.criteria.dependencies + case m: MultiMatchCriteria => m.dependencies case _ => Nil } @@ -285,10 +285,10 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { def valueAsString: String = maybeValue.map(v => s" $v").getOrElse("") override def sql = s"$identifier $notAsString$operator$valueAsString" - override def identifiers: Seq[Identifier] = + override lazy val dependencies: Seq[Identifier] = maybeValue match { - case Some(id: Identifier) => Seq(identifier, id) - case _ => Seq(identifier) + case Some(id: Identifier) => identifier.dependencies ++ id.dependencies + case _ => identifier.dependencies } override def extractAllMetricsPath: Map[String, String] = @@ -768,7 +768,7 @@ case class DistanceCriteria( } case class MultiMatchCriteria( - override val identifiers: Seq[Identifier], + identifiers: Seq[Identifier], value: StringValue, nestedElement: Option[NestedElement] = None ) extends Criteria { // FIXME map to multi_match @@ -778,6 +778,8 @@ case class MultiMatchCriteria( override def update(request: SingleSearch): Criteria = this.copy(identifiers = identifiers.map(_.update(request))) + override def dependencies: Seq[Identifier] = identifiers.flatMap(_.dependencies) + override lazy val nested: Boolean = identifiers.forall(_.nested) @tailrec diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index 4b18a3d4..cf6d9d3d 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -20,6 +20,7 @@ import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} import app.softnetwork.elastic.sql.schema.{ sqlConfig, Column, + Frequency, IngestPipeline, IngestPipelineType, IngestProcessor, @@ -27,16 +28,17 @@ import app.softnetwork.elastic.sql.schema.{ PartitionDate, RemoveProcessor, RenameProcessor, - Schema, ScriptProcessor, SetProcessor, - Table => DdlTable + Table => Schema, + TableType, + TransformTimeUnit } import app.softnetwork.elastic.sql.function.aggregate.WindowFunction import app.softnetwork.elastic.sql.serialization._ import com.fasterxml.jackson.databind.JsonNode -import java.time.Instant +import java.time.{Duration, Instant} package object query { sealed trait Statement extends Token @@ -561,6 +563,103 @@ package object query { sealed trait TableStatement extends DdlStatement + sealed trait MaterializedViewStatement extends TableStatement + + case class CreateMaterializedView( + view: String, + dql: DqlStatement, + ifNotExists: Boolean = false, + orReplace: Boolean = false, + frequency: Option[Frequency] = None, + options: Map[String, Value[_]] = Map.empty + ) extends MaterializedViewStatement { + override def sql: String = { + val frequencySql = frequency match { + case Some(freq) => freq.sql + case None => "" + } + val optionsSql = if (options.nonEmpty) { + s" WITH (${options + .map { case (k, v) => s"$k = $v" } + .mkString(", ")})" + } else { + "" + } + val replaceClause = if (orReplace) " OR REPLACE" else "" + val ineClause = if (!orReplace && ifNotExists) " IF NOT EXISTS" else "" + s"CREATE$replaceClause MATERIALIZED VIEW$ineClause $view$frequencySql$optionsSql AS ${dql.sql}" + } + + lazy val search: SingleSearch = dql match { + case s: SingleSearch => s + case _ => throw new IllegalArgumentException("Materialized view must be a single search") + } + + lazy val userLatency: Duration = options.get("user_latency") match { + case Some(value) => + value match { + case s: StringValue => + val regex = """\d+\s+(ms|s|m|h|d|w|M|y)""".r + regex.findFirstIn(s.value) match { + case Some(str) => + val parts = str.trim.split("\\s+") + val unit = parts(1) match { + case "ms" => TransformTimeUnit.Milliseconds + case "s" => TransformTimeUnit.Seconds + case "m" => TransformTimeUnit.Minutes + case "h" => TransformTimeUnit.Hours + case "d" => TransformTimeUnit.Days + case "w" => TransformTimeUnit.Weeks + case "M" => TransformTimeUnit.Months + case "y" => TransformTimeUnit.Years + case _ => TransformTimeUnit.Seconds + } + Duration.ofSeconds(Frequency(unit, parts(0).toInt).toSeconds) + case None => Duration.ofSeconds(5) + } + case i: LongValue => Duration.ofSeconds(i.value) + case _ => Duration.ofSeconds(5) + } + case None => Duration.ofSeconds(5) + } + } + + case class DropMaterializedView(name: String, ifExists: Boolean = false) + extends MaterializedViewStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) "IF EXISTS " else "" + s"DROP MATERIALIZED VIEW $ifExistsClause$name" + } + } + + case class RefreshMaterializedView(name: String, ifExists: Boolean = false) + extends MaterializedViewStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) "IF EXISTS " else "" + s"REFRESH MATERIALIZED VIEW $ifExistsClause$name" + } + } + + case class ShowMaterializedViewStatus(name: String) extends MaterializedViewStatement { + override def sql: String = s"SHOW MATERIALIZED VIEW STATUS $name" + } + + case class ShowMaterializedView(name: String) extends MaterializedViewStatement { + override def sql: String = s"SHOW MATERIALIZED VIEW $name" + } + + case object ShowMaterializedViews extends MaterializedViewStatement { + override def sql: String = s"SHOW MATERIALIZED VIEWS" + } + + case class ShowCreateMaterializedView(name: String) extends MaterializedViewStatement { + override def sql: String = s"SHOW CREATE MATERIALIZED VIEW $name" + } + + case class DescribeMaterializedView(name: String) extends MaterializedViewStatement { + override def sql: String = s"DESCRIBE MATERIALIZED VIEW $name" + } + case class CreateTable( table: String, ddl: Either[DqlStatement, List[Column]], @@ -645,14 +744,39 @@ package object query { case None => Map.empty } - lazy val schema: Schema = DdlTable( + lazy val materializedViews: List[String] = options.get("materialized_views") match { + case Some(value) => + value match { + case ov: StringValues => + ov.values.flatMap { + case o: StringValue => + Some(o.value) + case _ => None + }.toList + case _ => Nil + } + case None => Nil + } + + lazy val tableType: TableType = (options.get("type") match { + case Some(value) => + value match { + case s: StringValue => Some(TableType(s.value)) + case _ => None + } + case None => None + }).getOrElse(TableType.Regular) + + lazy val schema: Schema = Schema( name = table, columns = columns.toList, primaryKey = primaryKey, partitionBy = partitionBy, mappings = mappings, settings = settings, - aliases = aliases + aliases = aliases, + materializedViews = materializedViews, + tableType = tableType ).update() lazy val defaultPipeline: IngestPipeline = schema.defaultPipeline @@ -889,6 +1013,10 @@ package object query { override def sql: String = s"SHOW TABLE $table" } + case object ShowTables extends TableStatement { + override def sql: String = s"SHOW TABLES" + } + case class ShowCreateTable(table: String) extends TableStatement { override def sql: String = s"SHOW CREATE TABLE $table" } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 5b2ebf93..e9b21fa1 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -18,11 +18,19 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.config.ElasticSqlConfig +import app.softnetwork.elastic.sql.function.aggregate.{ + AggregateFunction, + AvgAgg, + CountAgg, + MaxAgg, + MinAgg, + SumAgg +} import app.softnetwork.elastic.sql.query._ import app.softnetwork.elastic.sql.serialization._ import app.softnetwork.elastic.sql.time.TimeUnit import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} -import com.fasterxml.jackson.databind.node.ObjectNode +import com.fasterxml.jackson.databind.node.{NullNode, ObjectNode} import java.util.UUID import scala.jdk.CollectionConverters._ @@ -429,7 +437,14 @@ package object schema { ) extends IngestProcessor { def processorType: IngestProcessorType = IngestProcessorType.Set + def isDefault: Boolean = copyFrom.isEmpty && value != Null && (doIf match { + case Some(i) if i.contains(s"ctx.$column == null") => true + case _ => false + }) + override def sql: String = { + if (isDefault) + return s"$column SET DEFAULT ${value.sql}" val base = copyFrom match { case Some(source) => s"$column COPY FROM $source" case None => s"$column SET VALUE ${value.sql}" @@ -578,6 +593,7 @@ package object schema { val processorsNode = mapper.createArrayNode() processors.foreach { processor => processorsNode.add(processor.node) + () } node.put("description", sql) node.set("processors", processorsNode) @@ -662,6 +678,8 @@ package object schema { ) } } + + def describe: Seq[Map[String, Any]] = processors.map(_.properties) } object IngestPipeline { @@ -688,6 +706,842 @@ package object schema { } } + sealed trait EnrichPolicyType { + def name: String + override def toString: String = name + } + + object EnrichPolicyType { + case object Match extends EnrichPolicyType { + val name: String = "MATCH" + } + case object GeoMatch extends EnrichPolicyType { + val name: String = "GEO_MATCH" + } + case object Range extends EnrichPolicyType { + val name: String = "RANGE" + } + } + + case class EnrichPolicy( + name: String, + policyType: EnrichPolicyType = EnrichPolicyType.Match, + indices: Seq[String], + matchField: String, + enrichFields: List[String], + criteria: Option[Criteria] = None + ) extends DdlToken { + def sql: String = + s"CREATE OR REPLACE ENRICH POLICY $name TYPE $policyType WITH SOURCE INDICES ${indices + .mkString(",")} MATCH FIELD $matchField ENRICH FIELDS (${enrichFields.mkString(", ")})${Where(criteria)}" + + def node(implicit criteriaToNode: Criteria => JsonNode): JsonNode = { + val node = mapper.createObjectNode() + node.put("name", name) + node.put("type", policyType.name) + val indicesNode = mapper.createArrayNode() + indices.foreach { index => + indicesNode.add(index) + () + } + node.set("indices", indicesNode) + node.put("match_field", matchField) + val enrichFieldsNode = mapper.createArrayNode() + enrichFields.foreach { field => + enrichFieldsNode.add(field) + () + } + node.set("enrich_fields", enrichFieldsNode) + criteria.foreach { c => + node.set("query", implicitly[JsonNode](c)) + } + node + } + } + + // ==================== Transform ==================== + + sealed trait TransformTimeUnit extends DdlToken { + def name: String + def sql: String = name + def format: String + } + + object TransformTimeUnit { + case object Milliseconds extends TransformTimeUnit { + val name: String = "MILLISECONDS" + override def format: String = "ms" + } + + case object Seconds extends TransformTimeUnit { + val name: String = "SECONDS" + override def format: String = "s" + } + + case object Minutes extends TransformTimeUnit { + val name: String = "MINUTES" + override def format: String = "m" + } + + case object Hours extends TransformTimeUnit { + val name: String = "HOURS" + override def format: String = "h" + } + + case object Days extends TransformTimeUnit { + val name: String = "DAYS" + override def format: String = "d" + } + + case object Weeks extends TransformTimeUnit { + val name: String = "WEEKS" + override def format: String = "w" + } + + case object Months extends TransformTimeUnit { + val name: String = "MONTHS" + override def format: String = "M" + } + + case object Years extends TransformTimeUnit { + val name: String = "YEARS" + override def format: String = "y" + } + + def apply(name: String): TransformTimeUnit = name.toUpperCase() match { + case "MILLISECOND" | "MILLISECONDS" => Milliseconds + case "SECOND" | "SECONDS" => Seconds + case "MINUTE" | "MINUTES" => Minutes + case "HOUR" | "HOURS" => Hours + case "DAY" | "DAYS" => Days + case "WEEK" | "WEEKS" => Weeks + case "MONTH" | "MONTHS" => Months + case "YEAR" | "YEARS" => Years + case other => throw new IllegalArgumentException(s"Invalid delay unit: $other") + } + } + + sealed trait TransformTimeInterval extends DdlToken { + def interval: Int + def timeUnit: TransformTimeUnit + def toSeconds: Int = timeUnit match { + case TransformTimeUnit.Milliseconds => interval / 1000 + case TransformTimeUnit.Seconds => interval + case TransformTimeUnit.Minutes => interval * 60 + case TransformTimeUnit.Hours => interval * 3600 + case TransformTimeUnit.Days => interval * 86400 + case TransformTimeUnit.Weeks => interval * 604800 + case TransformTimeUnit.Months => interval * 2592000 + case TransformTimeUnit.Years => interval * 31536000 + } + def toTransformFormat: String = s"$interval${timeUnit.format}" + + } + + object TransformTimeInterval { + + /** Creates a time interval from seconds */ + def fromSeconds(seconds: Int): (TransformTimeUnit, Int) = { + if (seconds >= 31536000 && seconds % 31536000 == 0) { + (TransformTimeUnit.Years, seconds / 31536000) + } else if (seconds >= 2592000 && seconds % 2592000 == 0) { + (TransformTimeUnit.Months, seconds / 2592000) + } else if (seconds >= 604800 && seconds % 604800 == 0) { + (TransformTimeUnit.Weeks, seconds / 604800) + } else if (seconds >= 86400 && seconds % 86400 == 0) { + (TransformTimeUnit.Days, seconds / 86400) + } else if (seconds >= 3600 && seconds % 3600 == 0) { + (TransformTimeUnit.Hours, seconds / 3600) + } else if (seconds >= 60 && seconds % 60 == 0) { + (TransformTimeUnit.Minutes, seconds / 60) + } else { + (TransformTimeUnit.Seconds, seconds) + } + } + } + + case class Delay( + timeUnit: TransformTimeUnit, + interval: Int + ) extends TransformTimeInterval { + def sql: String = s"WITH DELAY $interval $timeUnit" + } + + object Delay { + val Default: Delay = Delay(TransformTimeUnit.Minutes, 1) + + def fromSeconds(seconds: Int): Delay = { + val timeInterval = TransformTimeInterval.fromSeconds(seconds) + Delay(timeInterval._1, timeInterval._2) + } + + /** Calculates optimal delay based on frequency and number of stages + * + * Formula: delay = frequency / (nb_stages * buffer_factor) + * + * This ensures the complete chain can refresh within the specified frequency. The buffer + * factor adds safety margin for processing time. + * + * @param frequency + * Desired refresh frequency + * @param nbStages + * Total number of stages (changelog + enrichment + aggregate) + * @param bufferFactor + * Safety factor (default 1.5) + * @return + * Optimal delay for each Transform, or error if constraints cannot be met + */ + def calculateOptimal( + frequency: Frequency, + nbStages: Int, + bufferFactor: Double = 1.5 + ): Either[String, Delay] = { + if (nbStages <= 0) { + return Left("Number of stages must be positive") + } + + val frequencySeconds = frequency.toSeconds + val optimalDelaySeconds = (frequencySeconds / (nbStages * bufferFactor)).toInt + + // Validate constraints + if (optimalDelaySeconds < 10) { + Left( + s"Calculated delay ($optimalDelaySeconds seconds) is too small. " + + s"Consider increasing frequency or reducing number of stages. " + + s"Minimum required frequency: ${nbStages * bufferFactor * 10} seconds" + ) + } else if (optimalDelaySeconds > frequencySeconds / 2) { + Left( + s"Calculated delay ($optimalDelaySeconds seconds) is too large. " + + s"Each stage needs at least delay × 2 = frequency." + ) + } else { + Right(Delay.fromSeconds(optimalDelaySeconds)) + } + } + + /** Validates that a chain of transforms can refresh within the given frequency + * + * Requirements: + * - Total latency (delay × nbStages) must be less than frequency + * - Each transform runs every (delay × 2), so: delay × 2 × nbStages ≤ frequency + * + * @param delay + * Delay for each transform + * @param frequency + * Target refresh frequency + * @param nbStages + * Number of stages in the chain + * @return + * Success or error message + */ + def validate( + delay: Delay, + frequency: Frequency, + nbStages: Int + ): Either[String, Unit] = { + val totalLatency = delay.toSeconds * nbStages + val frequencySeconds = frequency.toSeconds + val requiredFrequency = delay.toSeconds * 2 * nbStages + + if (totalLatency > frequencySeconds) { + Left( + s"Total latency ($totalLatency seconds) exceeds frequency ($frequencySeconds seconds). " + + s"Minimum required frequency: $requiredFrequency seconds" + ) + } else if (requiredFrequency > frequencySeconds) { + Left( + s"Frequency ($frequencySeconds seconds) is too low for $nbStages stages with delay ${delay.toSeconds} seconds. " + + s"Minimum required frequency: $requiredFrequency seconds" + ) + } else { + Right(()) + } + } + } + + case class Frequency( + timeUnit: TransformTimeUnit, + interval: Int + ) extends TransformTimeInterval { + def sql: String = s"REFRESH EVERY $interval $timeUnit" + } + + case object Frequency { + val Default: Frequency = apply(Delay.Default) + def apply(delay: Delay): Frequency = Frequency(delay.timeUnit, delay.interval * 2) + def fromSeconds(seconds: Int): Frequency = { + val timeInterval = TransformTimeInterval.fromSeconds(seconds) + Frequency(timeInterval._1, timeInterval._2) + } + } + + case class TransformSource( + index: Seq[String], + query: Option[Criteria] + ) extends DdlToken { + override def sql: String = { + val queryStr = query.map(q => s" WHERE ${q.sql}").getOrElse("") + s"INDEX (${index.mkString(", ")})$queryStr" + } + + /** Converts to JSON for Elasticsearch + */ + def node(implicit criteriaToNode: Criteria => JsonNode): JsonNode = { + val node = mapper.createObjectNode() + val indicesNode = mapper.createArrayNode() + index.foreach(indicesNode.add) + node.set("index", indicesNode) + query.foreach { q => + node.set("query", implicitly[JsonNode](q)) + () + } + node + } + } + + case class TransformDest( + index: String, + pipeline: Option[String] = None + ) extends DdlToken { + override def sql: String = { + val pipelineStr = pipeline.map(p => s" PIPELINE $p").getOrElse("") + s"INDEX $index$pipelineStr" + } + def node: JsonNode = { + val node = mapper.createObjectNode() + node.put("index", index) + pipeline.foreach(p => node.put("pipeline", p)) + node + } + } + + /** Configuration for bucket selector (HAVING clause) + */ + case class TransformBucketSelectorConfig( + name: String = "having_filter", + bucketsPath: Map[String, String], + script: String + ) { + def node: JsonNode = { + val node = mapper.createObjectNode() + val bucketSelectorNode = mapper.createObjectNode() + val bucketsPathNode = mapper.createObjectNode() + bucketsPath.foreach { case (k, v) => + bucketsPathNode.put(k, v) + () + } + bucketSelectorNode.set("buckets_path", bucketsPathNode) + if (script.nonEmpty) { + bucketSelectorNode.put("script", script) + } + node.set("bucket_selector", bucketSelectorNode) + node + } + } + + case class TransformPivot( + groupBy: Map[String, TransformGroupBy], + aggregations: Map[String, TransformAggregation], + bucketSelector: Option[TransformBucketSelectorConfig] = None, + script: Option[String] = None + ) extends DdlToken { + override def sql: String = { + val groupByStr = groupBy + .map { case (name, gb) => + s"$name BY ${gb.sql}" + } + .mkString(", ") + + val aggStr = aggregations + .map { case (name, agg) => + s"$name AS ${agg.sql}" + } + .mkString(", ") + + val havingStr = bucketSelector.map(bs => s" HAVING ${bs.script}").getOrElse("") + + s"PIVOT ($groupByStr) AGGREGATE ($aggStr)$havingStr" + } + + /** Converts to JSON for Elasticsearch + */ + def node: JsonNode = { + val node = mapper.createObjectNode() + + val groupByNode = mapper.createObjectNode() + groupBy.foreach { case (name, gb) => + groupByNode.set(name, gb.node) + () + } + node.set("group_by", groupByNode) + + val aggsNode = mapper.createObjectNode() + aggregations.foreach { case (name, agg) => + aggsNode.set(name, agg.node) + () + } + bucketSelector.foreach { bs => + aggsNode.set(bs.name, bs.node) + () + } + node.set("aggregations", aggsNode) + + node + } + } + + sealed trait TransformGroupBy extends DdlToken { + def node: JsonNode + } + + case class TermsGroupBy(field: String) extends TransformGroupBy { + override def sql: String = s"TERMS($field)" + + override def node: JsonNode = { + val node = mapper.createObjectNode() + val termsNode = mapper.createObjectNode() + termsNode.put("field", field) + node.set("terms", termsNode) + node + } + } + + sealed trait TransformAggregation extends DdlToken { + def name: String + + def field: String + + override def sql: String = s"${name.toUpperCase}($field)" + + def node: JsonNode = { + val node = mapper.createObjectNode() + val fieldNode = mapper.createObjectNode() + fieldNode.put("field", field) + node.set(name.toLowerCase(), fieldNode) + node + } + } + + case class MaxTransformAggregation(field: String) extends TransformAggregation { + override def name: String = "max" + } + + case class MinTransformAggregation(field: String) extends TransformAggregation { + override def name: String = "min" + } + + case class SumTransformAggregation(field: String) extends TransformAggregation { + override def name: String = "sum" + } + + case class AvgTransformAggregation(field: String) extends TransformAggregation { + override def name: String = "avg" + } + + case class CountTransformAggregation(field: String) extends TransformAggregation { + override def name: String = "value_count" + + override def sql: String = if (field == "_id") "COUNT(*)" else s"COUNT($field)" + } + + case class CardinalityTransformAggregation(field: String) extends TransformAggregation { + override def name: String = "cardinality" + + override def sql: String = s"COUNT(DISTINCT $field)" + } + + case class TopHitsTransformAggregation( + fields: Seq[String], + size: Int = 1, + sort: Seq[FieldSort] = Seq( + FieldSort(Identifier(sqlConfig.transformLastUpdatedColumnName), Some(Desc)) + ) + ) extends TransformAggregation { + override def field: String = fields.mkString(", ") + + override def name: String = "top_hits" + + override def sql: String = { + val fieldsStr = fields.mkString(", ") + val sortStr = sort + .map { s => + s.sql + } + .mkString(", ") + s"TOP_HITS($fieldsStr) SIZE $size SORT BY ($sortStr)" + } + + override def node: JsonNode = { + val node = mapper.createObjectNode() + val topHitsNode = mapper.createObjectNode() + topHitsNode.put("size", size) + + // Sort configuration + val sortArray = mapper.createArrayNode() + sort.foreach { sortField => + val sortObj = mapper.createObjectNode() + val sortFieldObj = mapper.createObjectNode() + sortFieldObj.put("order", sortField.order.getOrElse(Desc).sql) + sortObj.set(sortField.name, sortFieldObj) + sortArray.add(sortObj) + } + topHitsNode.set("sort", sortArray) + + // Source filtering to include only the desired field + val sourceObj = mapper.createObjectNode() + val includesArray = mapper.createArrayNode() + fields.foreach { field => + includesArray.add(field) + () + } + sourceObj.set("includes", includesArray) + topHitsNode.set("_source", sourceObj) + + node.set("top_hits", topHitsNode) + node + } + } + + /** Extension methods for AggregateFunction + */ + implicit class AggregateConversion(agg: AggregateFunction) { + + /** Converts SQL aggregate function to Elasticsearch aggregation + */ + def toTransformAggregation: Option[TransformAggregation] = agg match { + case ma: MaxAgg => + Some(MaxTransformAggregation(ma.identifier.name)) + + case ma: MinAgg => + Some(MinTransformAggregation(ma.identifier.name)) + + case sa: SumAgg => + Some(SumTransformAggregation(sa.identifier.name)) + + case aa: AvgAgg => + Some(AvgTransformAggregation(aa.identifier.name)) + + case ca: CountAgg => + val field = ca.identifier.name + Some( + if (field == "*" || field.isEmpty) + if (ca.isCardinality) CardinalityTransformAggregation("_id") + else CountTransformAggregation("_id") + else if (ca.isCardinality) CardinalityTransformAggregation(field) + else CountTransformAggregation(field) + ) + + case _ => + // For other aggregate functions, default to none + None + } + } + + object ChangelogAggregationStrategy { + + /** Selects the appropriate aggregation for a field in a changelog based on its data type + * + * @param field + * The field to aggregate + * @param dataType + * The SQL data type of the field + * @return + * The transform aggregation to use + */ + def selectAggregation(field: String, dataType: SQLType): TransformAggregation = { + dataType match { + // Numeric types: Use MAX directly + case SQLTypes.Int | SQLTypes.BigInt | SQLTypes.Double | SQLTypes.Real => + MaxTransformAggregation(field) + + // Date/Timestamp: Use MAX directly + case SQLTypes.Date | SQLTypes.Timestamp => + MaxTransformAggregation(field) + + // Boolean: Use Top Hits + case SQLTypes.Boolean => + TopHitsTransformAggregation(Seq(field)) + + // Keyword: Already a keyword type, use MAX directly + case SQLTypes.Keyword => + TopHitsTransformAggregation(Seq(field)) + + // Text/Varchar: Analyzed field, must use .keyword multi-field + case SQLTypes.Text | SQLTypes.Varchar => + TopHitsTransformAggregation(Seq(s"$field.keyword")) // ✅ Use .keyword multi-field + + // For complex types, use top_hits as fallback + case _ => + TopHitsTransformAggregation(Seq(field)) + } + } + + /** Determines if a field can use simple MAX aggregation (without needing a multi-field) + */ + def canUseMaxDirectly(dataType: SQLType): Boolean = dataType match { + case SQLTypes.Int | SQLTypes.BigInt | SQLTypes.Double | SQLTypes.Real | SQLTypes.Date | + SQLTypes.Timestamp | SQLTypes.Keyword | SQLTypes.Boolean => + true + case _ => false + } + + /** Determines if a field needs a .keyword multi-field for aggregation + */ + def needsKeywordMultiField(dataType: SQLType): Boolean = dataType match { + case SQLTypes.Text | SQLTypes.Varchar => true + case _ => false + } + } + + case class TransformSync(time: TransformTimeSync) extends DdlToken { + override def sql: String = s"SYNC ${time.sql}" + + def node: JsonNode = { + val node = mapper.createObjectNode() + node.set("time", time.node) + node + } + } + + case class TransformTimeSync(field: String, delay: Delay) extends DdlToken { + override def sql: String = s"TIME FIELD $field ${delay.sql}" + + def node: JsonNode = { + val node = mapper.createObjectNode() + node.put("field", field) + node.put("delay", delay.toTransformFormat) + node + } + } + + case class TransformCreationStatus( + created: Boolean, + started: Option[Boolean] = None // None = not asked, Some(true/false) = result of start request + ) + + case class TransformLatest( + uniqueKey: Seq[String], + sort: String + ) extends DdlToken { + override def sql: String = { + val uniqueKeyStr = uniqueKey.mkString(", ") + s"LATEST UNIQUE KEY ($uniqueKeyStr) SORT BY $sort" + } + + def node: JsonNode = { + val node = mapper.createObjectNode() + val uniqueKeyNode = mapper.createArrayNode() + uniqueKey.foreach { key => + uniqueKeyNode.add(key) + () + } + node.set("unique_key", uniqueKeyNode) + node.put("sort", sort) + node + } + } + + /** Transform configuration models with built-in JSON serialization + */ + case class TransformConfig( + id: String, + viewName: String, + source: TransformSource, + dest: TransformDest, + pivot: Option[TransformPivot] = None, + latest: Option[TransformLatest] = None, + sync: Option[TransformSync] = None, + delay: Delay, + frequency: Frequency, + metadata: Map[String, AnyRef] = Map.empty + ) extends DdlToken { + + /** Delay in seconds for display */ + lazy val delaySeconds: Int = delay.toSeconds + + /** Frequency in seconds for display */ + lazy val frequencySeconds: Int = frequency.toSeconds + + /** Converts to Elasticsearch JSON format + */ + def node(implicit criteriaToNode: Criteria => JsonNode): JsonNode = { + val node = mapper.createObjectNode() + node.set("source", source.node) + node.set("dest", dest.node) + node.put("frequency", frequency.toTransformFormat) + sync.foreach { s => + node.set("sync", s.node) + () + } + pivot.foreach { p => + node.set("pivot", p.node) + () + } + latest.foreach { l => + node.set("latest", l.node) + () + } + node + } + + /** SQL DDL representation + */ + override def sql: String = { + val sb = new StringBuilder() + + sb.append(s"CREATE TRANSFORM $id\n") + sb.append(s" SOURCE ${source.sql}\n") + sb.append(s" DEST ${dest.sql}\n") + + pivot.foreach { p => + sb.append(s" ${p.sql}\n") + } + + sync.foreach { s => + sb.append(s" ${s.sql}\n") + } + + sb.append(s" ${frequency.sql}\n") + sb.append(s" ${delay.sql}") + + sb.toString() + } + + /** Human-readable description + */ + def description: String = { + s"Transform $id: ${source.index.mkString(", ")} → ${dest.index} " + + s"(every ${frequencySeconds}s, delay ${delaySeconds}s) for view $viewName" + } + } + + /** Health status + */ + sealed trait HealthStatus extends DdlToken { + def name: String + def sql: String = name + } + + object HealthStatus { + case object Green extends HealthStatus { + val name: String = "GREEN" + } + case object Yellow extends HealthStatus { + val name: String = "YELLOW" + } + case object Red extends HealthStatus { + val name: String = "RED" + } + + def apply(name: String): HealthStatus = name.toUpperCase() match { + case "GREEN" => Green + case "YELLOW" => Yellow + case "RED" => Red + case other => throw new IllegalArgumentException(s"Invalid health status: $other") + } + } + + /** Transform state + */ + sealed trait TransformState extends DdlToken { + def name: String + def sql: String = name + } + + object TransformState { + case object Started extends TransformState { + val name: String = "STARTED" + } + case object Indexing extends TransformState { + val name: String = "INDEXING" + } + case object Aborting extends TransformState { + val name: String = "ABORTING" + } + case object Stopping extends TransformState { + val name: String = "STOPPING" + } + case object Stopped extends TransformState { + val name: String = "STOPPED" + } + case object Failed extends TransformState { + val name: String = "FAILED" + } + case object Waiting extends TransformState { + val name: String = "WAITING" + } + case class Other(name: String) extends TransformState + + def apply(name: String): TransformState = name.toUpperCase() match { + case "STARTED" => Started + case "INDEXING" => Indexing + case "ABORTING" => Aborting + case "STOPPING" => Stopping + case "STOPPED" => Stopped + case "FAILED" => Failed + case "WAITING" => Waiting + case other => Other(other) + } + } + + /** Statistics for a Transform + * + * @param id + * Transform ID + * @param state + * Current state of the Transform + * @param documentsProcessed + * Total number of documents processed + * @param documentsIndexed + * Total number of documents indexed + * @param indexFailures + * Total number of indexing failures + * @param searchFailures + * Total number of search failures + * @param lastCheckpoint + * Last checkpoint number (if any) + * @param operationsBehind + * Number of operations behind source index + * @param processingTimeMs + * Total processing time in milliseconds + */ + case class TransformStats( + id: String, + state: TransformState, // "started", "stopped", "failed" + documentsProcessed: Long, + documentsIndexed: Long, + indexFailures: Long, + searchFailures: Long, + lastCheckpoint: Option[Long], + operationsBehind: Long, + processingTimeMs: Long + ) + // ==================== Schema ==================== + + /** Definition of a column within a table + * + * @param name + * the column name + * @param dataType + * the column SQL type + * @param script + * optional script processor associated to this column + * @param multiFields + * optional multi fields associated to this column + * @param defaultValue + * optional default value for this column + * @param notNull + * whether this column is not null + * @param comment + * optional comment for this column + * @param options + * optional options for this column (search analyzer, ...) + * @param struct + * optional parent struct column + * @param lineage + * sequence of (table, column) pairs indicating the lineage of this column + */ case class Column( name: String, dataType: SQLType, @@ -697,7 +1551,8 @@ package object schema { notNull: Boolean = false, comment: Option[String] = None, options: Map[String, Value[_]] = Map.empty, - struct: Option[Column] = None + struct: Option[Column] = None, + lineage: Map[String, Seq[(String, String)]] = Map.empty // ✅ Key = path ID, Value = chain ) extends DdlToken { def path: String = struct.map(st => s"${st.name}.$name").getOrElse(name) private def level: Int = struct.map(_.level + 1).getOrElse(0) @@ -714,6 +1569,18 @@ package object schema { } } + /** Flattens all lineage paths into a set of (table, column) pairs */ + def allLineageSources: Set[(String, String)] = + lineage.values.flatten.toSet + + /** Gets the immediate sources (last element of each path) */ + def immediateSources: Set[(String, String)] = + lineage.values.flatMap(_.lastOption).toSet + + /** Gets the original sources (first element of each path) */ + def originalSources: Set[(String, String)] = + lineage.values.flatMap(_.headOption).toSet + def _meta: Map[String, Value[_]] = { Map( "data_type" -> StringValue(dataType.typeId), @@ -734,7 +1601,25 @@ package object schema { "multi_fields" -> ObjectValue( multiFields.map(field => field.name -> ObjectValue(field._meta)).toMap ) - ) + ) ++ (if (lineage.nonEmpty) { + // ✅ Lineage as map of paths + Map( + "lineage" -> ObjectValue( + lineage.map { case (pathId, chain) => + pathId -> ObjectValues( + chain.map { case (table, column) => + ObjectValue( + Map( + "table" -> StringValue(table), + "column" -> StringValue(column) + ) + ) + } + ) + } + ) + ) + } else Map.empty) } def updateStruct(): Column = { @@ -1129,6 +2014,42 @@ package object schema { } } + sealed trait TableType { + def name: String + } + + object TableType { + case object Regular extends TableType { + override def name: String = "regular" + } + case object External extends TableType { + override def name: String = "external" + } + case object Changelog extends TableType { + override def name: String = "changelog" + } + case object Enrichment extends TableType { + override def name: String = "enrichment" + } + case object View extends TableType { + override def name: String = "view" + } + case object MaterializedView extends TableType { + override def name: String = "materialized_view" + } + + def apply(name: String): TableType = + name.toLowerCase match { + case "regular" => Regular + case "external" => External + case "changelog" => Changelog + case "enrichment" => Enrichment + case "view" => View + case "materialized_view" => MaterializedView + case other => throw new Exception(s"Unknown table type: $other") + } + } + /** Definition of a table within the schema * * @param name @@ -1147,6 +2068,10 @@ package object schema { * optional list of ingest processors associated to this table (apart from column processors) * @param aliases * optional map of aliases associated to this table + * @param materializedViews + * optional list of materialized views associated to this table + * @param materializedView + * whether this table is a materialized view */ case class Table( name: String, @@ -1156,8 +2081,14 @@ package object schema { mappings: Map[String, Value[_]] = Map.empty, settings: Map[String, Value[_]] = Map.empty, processors: Seq[IngestProcessor] = Seq.empty, - aliases: Map[String, Value[_]] = Map.empty + aliases: Map[String, Value[_]] = Map.empty, + materializedViews: List[String] = Nil, + tableType: TableType = TableType.Regular ) extends DdlToken { + lazy val isRegular: Boolean = tableType == TableType.Regular + + lazy val isPartitioned: Boolean = partitionBy.isDefined + lazy val indexName: String = name.toLowerCase private[schema] lazy val cols: Map[String, Column] = columns.map(c => c.name -> c).toMap @@ -1187,7 +2118,13 @@ package object schema { ) ) .getOrElse(Map.empty) ++ - Map("columns" -> ObjectValue(cols.map { case (name, col) => name -> ObjectValue(col._meta) })) + Map( + "columns" -> ObjectValue(cols.map { case (name, col) => name -> ObjectValue(col._meta) }) + ) ++ Map( + "type" -> StringValue(tableType.name) + ) ++ Map( + "materialized_views" -> StringValues(materializedViews.map(StringValue)) + ) def update(): Table = { val updated = @@ -1197,13 +2134,19 @@ package object schema { "_meta" -> ObjectValue(updated.mappings.get("_meta") match { case Some(ObjectValue(value)) => - (value - "primary_key" - "partition_by" - "columns") ++ updated._meta + (value - "primary_key" - "partition_by" - "columns" - "materialized_views" - "type") ++ updated._meta case _ => updated._meta }) ) ) } + lazy val metadata: Map[String, Any] = + mappings.get("_meta") match { + case Some(ObjectValue(value)) => ObjectValue(value) + case _ => Map.empty + } + def sql: String = { val opts = if (mappings.nonEmpty || settings.nonEmpty) { @@ -1245,6 +2188,8 @@ package object schema { .toSeq ++ implicitly[Seq[IngestProcessor]](primaryKey) def merge(statements: Seq[AlterTableStatement]): Table = { + if (!isRegular) + throw new Exception(s"Cannot alter table $name of type ${tableType.name}") statements .foldLeft(this) { (table, statement) => statement match { @@ -1637,7 +2582,7 @@ package object schema { val template = mapper.createObjectNode() template.set("mappings", indexMappings) template.set("settings", indexSettings) - if (aliases.nonEmpty) { + if (indexAliases.nonEmpty) { val aliasesNode = mapper.createObjectNode() indexAliases.foreach { alias => aliasesNode.set(alias.alias, alias.node) @@ -1714,6 +2659,8 @@ package object schema { aliases = aliasDiffs.toList ) } + + def describe: Seq[Map[String, Any]] = columns.flatMap(_.asMap) } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala index 24a1df33..ea9775bf 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala @@ -90,6 +90,12 @@ package object serialization { node } + implicit def objectValueToMap(value: ObjectValue): Map[String, Any] = { + import JacksonConfig.{objectMapper => mapper} + val node: ObjectNode = value + mapper.convertValue(node, classOf[Map[String, Any]]) + } + private[sql] def updateNode(node: ObjectNode, updates: Map[String, Value[_]]): ObjectNode = { import JacksonConfig.{objectMapper => mapper} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index 1178abda..de590577 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -239,7 +239,9 @@ class ParserSpec extends AnyFlatSpec with Matchers { import Queries._ - "Parser" should "parse numerical EQ" in { + behavior of "Parser DQL" + + it should "parse numerical EQ" in { val result = Parser(numericalEq) result.toOption .map(_.sql) @@ -939,6 +941,8 @@ class ParserSpec extends AnyFlatSpec with Matchers { // --- DDL --- + behavior of "Parser DDL" + it should "parse CREATE TABLE if not exists" in { val sql = """CREATE TABLE IF NOT EXISTS users ( @@ -994,16 +998,16 @@ class ParserSpec extends AnyFlatSpec with Matchers { println(schema.defaultPipeline.ddl) val json = schema.defaultPipeline.json println(json) - json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET VALUE 'anonymous' IF ctx.name == null, age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at SET VALUE _ingest.timestamp IF ctx.ingested_at == null, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name SET VALUE 'anonymous' IF ctx.name == null","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at SET VALUE _ingest.timestamp IF ctx.ingested_at == null","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3","ignore_failure":true}},{"date_index_name":{"field":"birthdate","description":"PARTITION BY birthdate (MONTH)","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at SET DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name SET DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at SET DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3","ignore_failure":true}},{"date_index_name":{"field":"birthdate","description":"PARTITION BY birthdate (MONTH)","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" val indexMappings = schema.indexMappings println(indexMappings) - indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer"},"name":{"type":"text","fields":{"raw":{"type":"keyword"}},"analyzer":"french","search_analyzer":"french"},"birthdate":{"type":"date"},"age":{"type":"integer"},"ingested_at":{"type":"date"},"profile":{"type":"object","properties":{"bio":{"type":"text"},"followers":{"type":"integer"},"join_date":{"type":"date"},"seniority":{"type":"integer"}}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"},"columns":{"age":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(birthdate, CURRENT_DATE, YEAR)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3"}},"profile":{"data_type":"STRUCT","not_null":"false","comment":"user profile","multi_fields":{"bio":{"data_type":"VARCHAR","not_null":"false"},"followers":{"data_type":"INT","not_null":"false"},"join_date":{"data_type":"DATE","not_null":"false"},"seniority":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3"}}}},"name":{"data_type":"VARCHAR","not_null":"false","default_value":"anonymous","multi_fields":{"raw":{"data_type":"KEYWORD","not_null":"false","comment":"sortable"}}},"ingested_at":{"data_type":"TIMESTAMP","not_null":"false","default_value":"_ingest.timestamp"},"birthdate":{"data_type":"DATE","not_null":"false"},"id":{"data_type":"INT","not_null":"true","comment":"user identifier"}}}}""".stripMargin + indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer"},"name":{"type":"text","fields":{"raw":{"type":"keyword"}},"analyzer":"french","search_analyzer":"french"},"birthdate":{"type":"date"},"age":{"type":"integer"},"ingested_at":{"type":"date"},"profile":{"type":"object","properties":{"bio":{"type":"text"},"followers":{"type":"integer"},"join_date":{"type":"date"},"seniority":{"type":"integer"}}}},"dynamic":false,"_meta":{"columns":{"age":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(birthdate, CURRENT_DATE, YEAR)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3"}},"profile":{"data_type":"STRUCT","not_null":"false","comment":"user profile","multi_fields":{"bio":{"data_type":"VARCHAR","not_null":"false"},"followers":{"data_type":"INT","not_null":"false"},"join_date":{"data_type":"DATE","not_null":"false"},"seniority":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3"}}}},"name":{"data_type":"VARCHAR","not_null":"false","default_value":"anonymous","multi_fields":{"raw":{"data_type":"KEYWORD","not_null":"false","comment":"sortable"}}},"ingested_at":{"data_type":"TIMESTAMP","not_null":"false","default_value":"_ingest.timestamp"},"birthdate":{"data_type":"DATE","not_null":"false"},"id":{"data_type":"INT","not_null":"true","comment":"user identifier"}},"materialized_views":[],"primary_key":["id"],"type":"regular","partition_by":{"column":"birthdate","granularity":"M"}}}""".stripMargin val indexSettings = schema.indexSettings println(indexSettings) indexSettings.toString shouldBe """{"index":{}}""" val pipeline = schema.defaultPipelineNode println(pipeline) - pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET VALUE 'anonymous' IF ctx.name == null, age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at SET VALUE _ingest.timestamp IF ctx.ingested_at == null, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name SET VALUE 'anonymous' IF ctx.name == null","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at SET VALUE _ingest.timestamp IF ctx.ingested_at == null","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3","ignore_failure":true}},{"date_index_name":{"field":"birthdate","description":"PARTITION BY birthdate (MONTH)","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at SET DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name SET DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at SET DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3","ignore_failure":true}},{"date_index_name":{"field":"birthdate","description":"PARTITION BY birthdate (MONTH)","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" // Reconstruct EsIndex val mappings = mapper.createObjectNode() mappings.set("mappings", indexMappings) @@ -1498,6 +1502,8 @@ class ParserSpec extends AnyFlatSpec with Matchers { // --- DML --- + behavior of "Parser DML" + it should "parse INSERT INTO ... VALUES" in { val sql = "INSERT INTO users (id, name) VALUES (1, 'Alice') ON CONFLICT DO NOTHING" val result = Parser(sql) @@ -1537,6 +1543,32 @@ class ParserSpec extends AnyFlatSpec with Matchers { } } + it should "parse INSERT INTO ... SELECT with alias" in { + val sql = """INSERT INTO orders_with_customers_mv_customers_changelog AS + |SELECT + | id, + | name, + | email, + | department.zipcode AS department.zip_code + |FROM customers; + |""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case Insert( + "orders_with_customers_mv_customers_changelog", + Nil, + Left(sel: DqlStatement), + None + ) => + sel.sql should include( + "SELECT id, name, email, department.zipcode AS department.zip_code FROM customers" + ) + case _ => fail("Expected Insert with select") + } + } + it should "parse INSERT INTO ... SELECT with ON CONFLICT" in { val sql = "INSERT INTO users SELECT id, name FROM old_users ON CONFLICT (id) DO UPDATE" val result = Parser(sql) @@ -1589,4 +1621,5 @@ class ParserSpec extends AnyFlatSpec with Matchers { case _ => fail("Expected Union") } } + } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala index ce8fa93b..c48cf40e 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala @@ -93,7 +93,8 @@ trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala def assertSelectResult( res: ElasticResult[QueryResult], - rows: Seq[Map[String, Any]] = Seq.empty + rows: Seq[Map[String, Any]] = Seq.empty, + nbResults: Option[Int] = None ): Unit = { res.isSuccess shouldBe true res.toOption.get match { @@ -106,6 +107,8 @@ trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala if (rows.nonEmpty) { results.size shouldBe rows.size results should contain theSameElementsAs rows + } else if (nbResults.isDefined) { + results.size shouldBe nbResults.get } else { log.info(s"Rows: $results") } @@ -115,6 +118,8 @@ trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala if (rows.nonEmpty) { results.size shouldBe rows.size results should contain theSameElementsAs rows + } else if (nbResults.isDefined) { + results.size shouldBe nbResults.get } else { log.info(s"Rows: $results") } @@ -123,6 +128,8 @@ trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala if (rows.nonEmpty) { results.size shouldBe rows.size results should contain theSameElementsAs rows + } else if (nbResults.isDefined) { + results.size shouldBe nbResults.get } else { log.info(s"Rows: $results") } @@ -156,6 +163,16 @@ trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala } } + // ------------------------------------------------------------------------- + // Helper: assert SHOW TABLES result type + // ------------------------------------------------------------------------- + + def assertShowTables(res: ElasticResult[QueryResult]): Seq[Map[String, Any]] = { + res.isSuccess shouldBe true + res.toOption.get shouldBe a[QueryRows] + res.toOption.get.asInstanceOf[QueryRows].rows + } + // ------------------------------------------------------------------------- // Helper: assert SHOW TABLE result type // ------------------------------------------------------------------------- @@ -638,6 +655,17 @@ trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala table.ddl should include("visibility BOOLEAN DEFAULT true") } + it should "list all tables" in { + val tables = assertShowTables(client.run("SHOW TABLES").futureValue) + tables should not be empty + for { + table <- tables + } { + table should contain key "name" + table should contain key "type" + } + } + // =========================================================================== // 3. DML — INSERT / UPDATE / DELETE // =========================================================================== diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala index 023bb907..10c726c1 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala @@ -22,6 +22,7 @@ import akka.stream.scaladsl.{Flow, Source} import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result.ElasticResult import app.softnetwork.elastic.client.scroll._ +import app.softnetwork.elastic.sql.PainlessContextType import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.sql.schema.TableAlias import app.softnetwork.serialization._ @@ -290,9 +291,10 @@ trait MockElasticClientApi extends NopeClientApi { // ==================== SearchApi ==================== - override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: SingleSearch - )(implicit timestamp: Long): String = + override private[client] implicit def singleSearchToJsonQuery(singleSearch: SingleSearch)(implicit + timestamp: Long, + contextType: PainlessContextType = PainlessContextType.Query + ): String = """{ | "query": { | "match_all": {}