diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 5c55fdcb..bf097a6e 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -572,6 +572,57 @@ } } }, + "@@envoy_api+//bazel:repositories.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "DITA13hLUg1ghuzFJXkoidwLUcSUZ8eP36QseHw3oQ4=", + "usagesDigest": "lOhJkV09ITWn6LOK9fLMuf1t3969wdr45lToQ2MVQoU=", + "recordedInputs": [ + "REPO_MAPPING:envoy_api+,bazel_tools bazel_tools", + "REPO_MAPPING:envoy_api+,envoy_api envoy_api+" + ], + "generatedRepoSpecs": { + "prometheus_metrics_model": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/prometheus/client_model/archive/v0.6.2.tar.gz" + ], + "sha256": "47c5ea7949f68e7f7b344350c59b6bd31eeb921f0eec6c3a566e27cf1951470c", + "strip_prefix": "client_model-0.6.2", + "build_file_content": "\nload(\"@envoy_api//bazel:api_build_system.bzl\", \"api_cc_py_proto_library\")\nload(\"@io_bazel_rules_go//proto:def.bzl\", \"go_proto_library\")\n\napi_cc_py_proto_library(\n name = \"client_model\",\n srcs = [\n \"io/prometheus/client/metrics.proto\",\n ],\n visibility = [\"//visibility:public\"],\n)\n\ngo_proto_library(\n name = \"client_model_go_proto\",\n importpath = \"github.com/prometheus/client_model/go\",\n proto = \":client_model\",\n visibility = [\"//visibility:public\"],\n)\n" + } + } + } + } + }, + "@@googleapis+//:extensions.bzl%switched_rules": { + "general": { + "bzlTransitiveDigest": "dGjdAR2OztzWMSyz8eMU1fObvEYPBQh2DdtbMY2PmNc=", + "usagesDigest": "phuibqqjsm5RJqquWNJ7BXJvnPPgSCBreyRLEH/JjIA=", + "recordedInputs": [], + "generatedRepoSpecs": {} + } + }, + "@@pybind11_bazel+//:internal_configure.bzl%internal_configure_extension": { + "general": { + "bzlTransitiveDigest": "vQXYYIISsqHYpUHoDTN9dB7+8Y13YPTb7fJltJA5d6Y=", + "usagesDigest": "tVQNvLoXMWAbiK39am3yovKGpwINdftfn7RpDyN+JZc=", + "recordedInputs": [ + "REPO_MAPPING:pybind11_bazel+,bazel_tools bazel_tools" + ], + "generatedRepoSpecs": { + "pybind11": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file": "@@pybind11_bazel+//:pybind11-BUILD.bazel", + "strip_prefix": "pybind11-2.13.6", + "url": "https://github.com/pybind/pybind11/archive/refs/tags/v2.13.6.tar.gz", + "integrity": "sha256-4Iy4f0dz2pf6e18DXeh2OrxlbYfVdz5i9toFh9Hw7CA=" + } + } + } + } + }, "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { "general": { "bzlTransitiveDigest": "m5FNZZeNCyu5NFVLxp9nxedZaJmJL7/GN/fgl03/YTM=", @@ -919,6 +970,69 @@ } } }, + "@@rules_perl+//perl:extensions.bzl%perl_repositories": { + "general": { + "bzlTransitiveDigest": "K8OybB84tBcly/FCawLeUWEaEGko6fQHirIx59UasjQ=", + "usagesDigest": "qSSNDdCNVxNhY36wMndEAFacdhR0ooLTmumfad0km9s=", + "recordedInputs": [ + "REPO_MAPPING:rules_perl+,bazel_tools bazel_tools", + "REPO_MAPPING:rules_perl+,rules_perl rules_perl+" + ], + "generatedRepoSpecs": { + "perl_darwin_arm64": { + "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", + "attributes": { + "strip_prefix": "perl-darwin-arm64", + "sha256": "285769f3c50c339fb59a3987b216ae3c5c573b95babe6875a1ef56fb178433da", + "urls": [ + "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-darwin-arm64.tar.xz" + ] + } + }, + "perl_darwin_amd64": { + "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", + "attributes": { + "strip_prefix": "perl-darwin-amd64", + "sha256": "63bc5ee36f5394d71c50cca6cafdd333ee58f9eaa40bca63c85f9bd06f2c1fd6", + "urls": [ + "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-darwin-amd64.tar.xz" + ] + } + }, + "perl_linux_amd64": { + "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", + "attributes": { + "strip_prefix": "perl-linux-amd64", + "sha256": "3bdffa9d7a3f97c0207314637b260ba5115b1d0829f97e3e2e301191a4d4d076", + "urls": [ + "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-linux-amd64.tar.xz" + ] + } + }, + "perl_linux_arm64": { + "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", + "attributes": { + "strip_prefix": "perl-linux-arm64", + "sha256": "6fa4ece99e790ecbc2861f6ecb7b52694c01c2eeb215b4370f16a3b12d952117", + "urls": [ + "https://github.com/skaji/relocatable-perl/releases/download/5.36.0.1/perl-linux-arm64.tar.xz" + ] + } + }, + "perl_windows_x86_64": { + "repoRuleId": "@@rules_perl+//perl:repo.bzl%perl_download", + "attributes": { + "strip_prefix": "", + "sha256": "aeb973da474f14210d3e1a1f942dcf779e2ae7e71e4c535e6c53ebabe632cc98", + "urls": [ + "https://mirror.bazel.build/strawberryperl.com/download/5.32.1.1/strawberry-perl-5.32.1.1-64bit.zip", + "https://strawberryperl.com/download/5.32.1.1/strawberry-perl-5.32.1.1-64bit.zip" + ] + } + } + } + } + }, "@@rules_python+//python/extensions:config.bzl%config": { "general": { "bzlTransitiveDigest": "tpiXvHXWP8z23XkccFfNPavHB4Sq2c8kxPlZAR2mN5A=", @@ -5601,6 +5715,105 @@ } } } + }, + "@@rules_rust+//crate_universe/private:internal_extensions.bzl%cu_nr": { + "general": { + "bzlTransitiveDigest": "DGMBY1dpvP/xuzm7me7Ny9YWY2PbbS18SWkGV04oHQA=", + "usagesDigest": "v4We18mWSPeKV4GPp9Gne78W+jZOgP2pC1i4UN9br1g=", + "recordedInputs": [ + "REPO_MAPPING:bazel_features+,bazel_features_globals bazel_features++version_extension+bazel_features_globals", + "REPO_MAPPING:bazel_features+,bazel_features_version bazel_features++version_extension+bazel_features_version", + "REPO_MAPPING:rules_cc+,bazel_skylib bazel_skylib+", + "REPO_MAPPING:rules_cc+,bazel_tools bazel_tools", + "REPO_MAPPING:rules_cc+,cc_compatibility_proxy rules_cc++compatibility_proxy+cc_compatibility_proxy", + "REPO_MAPPING:rules_cc+,platforms platforms", + "REPO_MAPPING:rules_cc+,rules_cc rules_cc+", + "REPO_MAPPING:rules_cc++compatibility_proxy+cc_compatibility_proxy,rules_cc rules_cc+", + "REPO_MAPPING:rules_rust+,bazel_features bazel_features+", + "REPO_MAPPING:rules_rust+,bazel_skylib bazel_skylib+", + "REPO_MAPPING:rules_rust+,bazel_tools bazel_tools", + "REPO_MAPPING:rules_rust+,cui rules_rust++cu+cui", + "REPO_MAPPING:rules_rust+,rrc rules_rust++i2+rrc", + "REPO_MAPPING:rules_rust+,rules_cc rules_cc+", + "REPO_MAPPING:rules_rust+,rules_rust rules_rust+" + ], + "generatedRepoSpecs": { + "cargo_bazel_bootstrap": { + "repoRuleId": "@@rules_rust+//cargo/private:cargo_bootstrap.bzl%cargo_bootstrap_repository", + "attributes": { + "srcs": [ + "@@rules_rust+//crate_universe:src/api.rs", + "@@rules_rust+//crate_universe:src/api/lockfile.rs", + "@@rules_rust+//crate_universe:src/cli.rs", + "@@rules_rust+//crate_universe:src/cli/generate.rs", + "@@rules_rust+//crate_universe:src/cli/query.rs", + "@@rules_rust+//crate_universe:src/cli/render.rs", + "@@rules_rust+//crate_universe:src/cli/splice.rs", + "@@rules_rust+//crate_universe:src/cli/vendor.rs", + "@@rules_rust+//crate_universe:src/config.rs", + "@@rules_rust+//crate_universe:src/context.rs", + "@@rules_rust+//crate_universe:src/context/crate_context.rs", + "@@rules_rust+//crate_universe:src/context/platforms.rs", + "@@rules_rust+//crate_universe:src/lib.rs", + "@@rules_rust+//crate_universe:src/lockfile.rs", + "@@rules_rust+//crate_universe:src/main.rs", + "@@rules_rust+//crate_universe:src/metadata.rs", + "@@rules_rust+//crate_universe:src/metadata/cargo_bin.rs", + "@@rules_rust+//crate_universe:src/metadata/cargo_tree_resolver.rs", + "@@rules_rust+//crate_universe:src/metadata/cargo_tree_rustc_wrapper.bat", + "@@rules_rust+//crate_universe:src/metadata/cargo_tree_rustc_wrapper.sh", + "@@rules_rust+//crate_universe:src/metadata/dependency.rs", + "@@rules_rust+//crate_universe:src/metadata/metadata_annotation.rs", + "@@rules_rust+//crate_universe:src/rendering.rs", + "@@rules_rust+//crate_universe:src/rendering/template_engine.rs", + "@@rules_rust+//crate_universe:src/rendering/templates/module_bzl.j2", + "@@rules_rust+//crate_universe:src/rendering/templates/partials/header.j2", + "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/aliases_map.j2", + "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/deps_map.j2", + "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/repo_git.j2", + "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/repo_http.j2", + "@@rules_rust+//crate_universe:src/rendering/templates/vendor_module.j2", + "@@rules_rust+//crate_universe:src/rendering/verbatim/alias_rules.bzl", + "@@rules_rust+//crate_universe:src/select.rs", + "@@rules_rust+//crate_universe:src/splicing.rs", + "@@rules_rust+//crate_universe:src/splicing/cargo_config.rs", + "@@rules_rust+//crate_universe:src/splicing/crate_index_lookup.rs", + "@@rules_rust+//crate_universe:src/splicing/splicer.rs", + "@@rules_rust+//crate_universe:src/test.rs", + "@@rules_rust+//crate_universe:src/utils.rs", + "@@rules_rust+//crate_universe:src/utils/starlark.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/glob.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/label.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/select.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/select_dict.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/select_list.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/select_scalar.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/select_set.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/serialize.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/target_compatible_with.rs", + "@@rules_rust+//crate_universe:src/utils/symlink.rs", + "@@rules_rust+//crate_universe:src/utils/target_triple.rs" + ], + "binary": "cargo-bazel", + "cargo_lockfile": "@@rules_rust+//crate_universe:Cargo.lock", + "cargo_toml": "@@rules_rust+//crate_universe:Cargo.toml", + "version": "1.86.0", + "timeout": 900, + "rust_toolchain_cargo_template": "@rust_host_tools//:bin/{tool}", + "rust_toolchain_rustc_template": "@rust_host_tools//:bin/{tool}", + "compressed_windows_toolchain_names": false + } + } + }, + "moduleExtensionMetadata": { + "explicitRootModuleDirectDeps": [ + "cargo_bazel_bootstrap" + ], + "explicitRootModuleDirectDevDeps": [], + "useAllRepos": "NO", + "reproducible": false + } + } } }, "facts": { diff --git a/bazel/java.MODULE.bazel b/bazel/java.MODULE.bazel index efa94121..1f82fe4a 100644 --- a/bazel/java.MODULE.bazel +++ b/bazel/java.MODULE.bazel @@ -15,6 +15,10 @@ maven.install( "com.fasterxml.jackson.core:jackson-annotations", "com.fasterxml.jackson.core:jackson-core", "com.fasterxml.jackson.core:jackson-databind", + "com.zaxxer:HikariCP:6.3.3", + "io.github.tors42:chariot:0.1.10", + "com.h2database:h2:2.2.224", + "org.postgresql:postgresql:42.7.4", "com.fasterxml.jackson.datatype:jackson-datatype-guava", "com.fasterxml.jackson.datatype:jackson-datatype-jdk8", "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", @@ -51,6 +55,10 @@ maven.install( "com.fasterxml.jackson:jackson-bom:2.21.0", ], generate_compat_repositories = True, + known_contributing_modules = [ + "moon-base", + "protobuf", + ], lock_file = "//:maven_install.json", repositories = [ "https://repo1.maven.org/maven2", diff --git a/jvm/src/main/java/com/muchq/indexer/App.java b/jvm/src/main/java/com/muchq/indexer/App.java new file mode 100644 index 00000000..8badfd2d --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/App.java @@ -0,0 +1,14 @@ +package com.muchq.indexer; + +import io.micronaut.runtime.Micronaut; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class App { + private static final Logger LOG = LoggerFactory.getLogger(App.class); + + public static void main(String[] args) { + LOG.info("Starting Chess Game Indexer"); + Micronaut.run(App.class, args); + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/BUILD.bazel b/jvm/src/main/java/com/muchq/indexer/BUILD.bazel new file mode 100644 index 00000000..f76c5c09 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/BUILD.bazel @@ -0,0 +1,50 @@ +load("@rules_java//java:java_binary.bzl", "java_binary") +load("//bazel/rules:oci.bzl", "linux_oci_java") + +java_binary( + name = "indexer", + srcs = [ + "App.java", + "IndexerModule.java", + ], + main_class = "com.muchq.indexer.App", + plugins = [ + "//bazel/rules:micronaut_type_element_visitor_processor", + "//bazel/rules:micronaut_aggregating_type_element_visitor_processor", + "//bazel/rules:micronaut_bean_definition_inject_processor", + "//bazel/rules:micronaut_package_element_visitor_processor", + ], + resources = ["//jvm/src/main/resources:micronaut_config"], + visibility = ["//visibility:public"], + runtime_deps = [ + "@maven//:ch_qos_logback_logback_classic", + "@maven//:org_postgresql_postgresql", + ], + deps = [ + "//jvm/src/main/java/com/muchq/chess_com_api", + "//jvm/src/main/java/com/muchq/http_client/core", + "//jvm/src/main/java/com/muchq/http_client/jdk11", + "//jvm/src/main/java/com/muchq/indexer/api", + "//jvm/src/main/java/com/muchq/indexer/api/dto", + "//jvm/src/main/java/com/muchq/indexer/chessql/compiler", + "//jvm/src/main/java/com/muchq/indexer/db", + "//jvm/src/main/java/com/muchq/indexer/engine", + "//jvm/src/main/java/com/muchq/indexer/engine/model", + "//jvm/src/main/java/com/muchq/indexer/motifs", + "//jvm/src/main/java/com/muchq/indexer/queue", + "//jvm/src/main/java/com/muchq/indexer/worker", + "//jvm/src/main/java/com/muchq/json", + "@maven//:com_fasterxml_jackson_core_jackson_databind", + "@maven//:com_zaxxer_HikariCP", + "@maven//:io_micronaut_jaxrs_micronaut_jaxrs_server", + "@maven//:io_micronaut_micronaut_context", + "@maven//:io_micronaut_micronaut_http_server_netty", + "@maven//:io_micronaut_micronaut_inject", + "@maven//:io_micronaut_micronaut_jackson_databind", + "@maven//:io_micronaut_micronaut_runtime", + "@maven//:jakarta_inject_jakarta_inject_api", + "@maven//:org_slf4j_slf4j_api", + ], +) + +linux_oci_java(bin_name = "indexer") diff --git a/jvm/src/main/java/com/muchq/indexer/IndexerModule.java b/jvm/src/main/java/com/muchq/indexer/IndexerModule.java new file mode 100644 index 00000000..4c37598f --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/IndexerModule.java @@ -0,0 +1,136 @@ +package com.muchq.indexer; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.muchq.chess_com_api.ChessClient; +import com.muchq.http_client.core.HttpClient; +import com.muchq.http_client.jdk.Jdk11HttpClient; +import com.muchq.indexer.chessql.compiler.CompiledQuery; +import com.muchq.indexer.chessql.compiler.QueryCompiler; +import com.muchq.indexer.chessql.compiler.SqlCompiler; +import com.muchq.indexer.db.DataSourceFactory; +import com.muchq.indexer.db.GameFeatureDao; +import com.muchq.indexer.db.GameFeatureStore; +import com.muchq.indexer.db.IndexingRequestDao; +import com.muchq.indexer.db.IndexingRequestStore; +import com.muchq.indexer.db.Migration; +import com.muchq.indexer.engine.FeatureExtractor; +import com.muchq.indexer.engine.GameReplayer; +import com.muchq.indexer.engine.PgnParser; +import com.muchq.indexer.motifs.CrossPinDetector; +import com.muchq.indexer.motifs.DiscoveredAttackDetector; +import com.muchq.indexer.motifs.ForkDetector; +import com.muchq.indexer.motifs.MotifDetector; +import com.muchq.indexer.motifs.PinDetector; +import com.muchq.indexer.motifs.SkewerDetector; +import com.muchq.indexer.queue.InMemoryIndexQueue; +import com.muchq.indexer.queue.IndexQueue; +import com.muchq.indexer.worker.IndexWorker; +import com.muchq.indexer.worker.IndexWorkerLifecycle; +import com.muchq.json.JsonUtils; +import io.micronaut.context.annotation.Context; +import io.micronaut.context.annotation.Factory; +import io.micronaut.context.annotation.Value; + +import javax.sql.DataSource; +import java.util.List; + +@Factory +public class IndexerModule { + + @Context + public ObjectMapper objectMapper() { + return JsonUtils.mapper(); + } + + @Context + public HttpClient httpClient() { + return new Jdk11HttpClient(java.net.http.HttpClient.newHttpClient()); + } + + @Context + public ChessClient chessClient(HttpClient httpClient, ObjectMapper objectMapper) { + return new ChessClient(httpClient, objectMapper); + } + + @Context + public DataSource dataSource( + @Value("${indexer.db.url:jdbc:h2:mem:indexer;DB_CLOSE_DELAY=-1}") String jdbcUrl, + @Value("${indexer.db.username:sa}") String username, + @Value("${indexer.db.password:}") String password) { + return DataSourceFactory.create(jdbcUrl, username, password); + } + + @Context + public Migration migration( + DataSource dataSource, + @Value("${indexer.db.url:jdbc:h2:mem:indexer;DB_CLOSE_DELAY=-1}") String jdbcUrl) { + boolean useH2 = jdbcUrl.contains(":h2:"); + Migration migration = new Migration(dataSource, useH2); + migration.run(); + return migration; + } + + @Context + public IndexingRequestStore indexingRequestStore(DataSource dataSource) { + return new IndexingRequestDao(dataSource); + } + + @Context + public GameFeatureStore gameFeatureStore( + DataSource dataSource, + @Value("${indexer.db.url:jdbc:h2:mem:indexer;DB_CLOSE_DELAY=-1}") String jdbcUrl) { + boolean useH2 = jdbcUrl.contains(":h2:"); + return new GameFeatureDao(dataSource, useH2); + } + + @Context + public IndexQueue indexQueue() { + return new InMemoryIndexQueue(); + } + + @Context + public QueryCompiler queryCompiler() { + return new SqlCompiler(); + } + + @Context + public List motifDetectors() { + return List.of( + new PinDetector(), + new CrossPinDetector(), + new ForkDetector(), + new SkewerDetector(), + new DiscoveredAttackDetector() + ); + } + + @Context + public PgnParser pgnParser() { + return new PgnParser(); + } + + @Context + public GameReplayer gameReplayer() { + return new GameReplayer(); + } + + @Context + public FeatureExtractor featureExtractor(PgnParser pgnParser, GameReplayer replayer, List detectors) { + return new FeatureExtractor(pgnParser, replayer, detectors); + } + + @Context + public IndexWorker indexWorker( + ChessClient chessClient, + FeatureExtractor featureExtractor, + IndexingRequestStore requestStore, + GameFeatureStore gameFeatureStore, + ObjectMapper objectMapper) { + return new IndexWorker(chessClient, featureExtractor, requestStore, gameFeatureStore, objectMapper); + } + + @Context + public IndexWorkerLifecycle indexWorkerLifecycle(IndexQueue queue, IndexWorker worker) { + return new IndexWorkerLifecycle(queue, worker); + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/api/BUILD.bazel b/jvm/src/main/java/com/muchq/indexer/api/BUILD.bazel new file mode 100644 index 00000000..aa795358 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/api/BUILD.bazel @@ -0,0 +1,28 @@ +load("@rules_java//java:java_library.bzl", "java_library") + +java_library( + name = "api", + srcs = [ + "IndexController.java", + "QueryController.java", + ], + plugins = [ + "//bazel/rules:micronaut_type_element_visitor_processor", + "//bazel/rules:micronaut_aggregating_type_element_visitor_processor", + "//bazel/rules:micronaut_bean_definition_inject_processor", + "//bazel/rules:micronaut_package_element_visitor_processor", + ], + visibility = ["//visibility:public"], + deps = [ + "//jvm/src/main/java/com/muchq/indexer/api/dto", + "//jvm/src/main/java/com/muchq/indexer/chessql/ast", + "//jvm/src/main/java/com/muchq/indexer/chessql/compiler", + "//jvm/src/main/java/com/muchq/indexer/chessql/parser", + "//jvm/src/main/java/com/muchq/indexer/db", + "//jvm/src/main/java/com/muchq/indexer/queue", + "@maven//:io_micronaut_jaxrs_micronaut_jaxrs_server", + "@maven//:jakarta_inject_jakarta_inject_api", + "@maven//:jakarta_ws_rs_jakarta_ws_rs_api", + "@maven//:org_slf4j_slf4j_api", + ], +) diff --git a/jvm/src/main/java/com/muchq/indexer/api/IndexController.java b/jvm/src/main/java/com/muchq/indexer/api/IndexController.java new file mode 100644 index 00000000..8536ee0d --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/api/IndexController.java @@ -0,0 +1,68 @@ +package com.muchq.indexer.api; + +import com.muchq.indexer.api.dto.IndexRequest; +import com.muchq.indexer.api.dto.IndexResponse; +import com.muchq.indexer.db.IndexingRequestStore; +import com.muchq.indexer.queue.IndexMessage; +import com.muchq.indexer.queue.IndexQueue; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import jakarta.inject.Singleton; +import java.util.UUID; + +@Singleton +@Path("/index") +public class IndexController { + private static final Logger LOG = LoggerFactory.getLogger(IndexController.class); + + private final IndexingRequestStore requestDao; + private final IndexQueue queue; + + public IndexController(IndexingRequestStore requestDao, IndexQueue queue) { + this.requestDao = requestDao; + this.queue = queue; + } + + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public IndexResponse createIndex(IndexRequest request) { + LOG.info("POST /index player={} platform={} months={}-{}", + request.player(), request.platform(), request.startMonth(), request.endMonth()); + + UUID id = requestDao.create( + request.player(), + request.platform(), + request.startMonth(), + request.endMonth() + ); + + queue.enqueue(new IndexMessage( + id, + request.player(), + request.platform(), + request.startMonth(), + request.endMonth() + )); + + return new IndexResponse(id, "PENDING", 0, null); + } + + @GET + @Path("/{id}") + @Produces(MediaType.APPLICATION_JSON) + public IndexResponse getIndex(@PathParam("id") UUID id) { + LOG.info("GET /index/{}", id); + return requestDao.findById(id) + .map(row -> new IndexResponse(row.id(), row.status(), row.gamesIndexed(), row.errorMessage())) + .orElseThrow(() -> new RuntimeException("Indexing request not found: " + id)); + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/api/QueryController.java b/jvm/src/main/java/com/muchq/indexer/api/QueryController.java new file mode 100644 index 00000000..f5f22d11 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/api/QueryController.java @@ -0,0 +1,53 @@ +package com.muchq.indexer.api; + +import com.muchq.indexer.api.dto.GameFeatureRow; +import com.muchq.indexer.api.dto.QueryRequest; +import com.muchq.indexer.api.dto.QueryResponse; +import com.muchq.indexer.chessql.ast.Expr; +import com.muchq.indexer.chessql.compiler.CompiledQuery; +import com.muchq.indexer.chessql.compiler.QueryCompiler; +import com.muchq.indexer.chessql.parser.Parser; +import com.muchq.indexer.db.GameFeatureStore; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import jakarta.inject.Singleton; +import java.util.List; + +@Singleton +@Path("/query") +public class QueryController { + private static final Logger LOG = LoggerFactory.getLogger(QueryController.class); + + private final GameFeatureStore gameFeatureStore; + private final QueryCompiler queryCompiler; + + public QueryController(GameFeatureStore gameFeatureStore, QueryCompiler queryCompiler) { + this.gameFeatureStore = gameFeatureStore; + this.queryCompiler = queryCompiler; + } + + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public QueryResponse query(QueryRequest request) { + LOG.info("POST /query query={} limit={} offset={}", request.query(), request.limit(), request.offset()); + + Expr expr = Parser.parse(request.query()); + CompiledQuery compiled = queryCompiler.compile(expr); + + List rows = + gameFeatureStore.query(compiled, request.limit(), request.offset()); + + List dtos = rows.stream() + .map(GameFeatureRow::fromStore) + .toList(); + + return new QueryResponse(dtos, dtos.size()); + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/api/dto/BUILD.bazel b/jvm/src/main/java/com/muchq/indexer/api/dto/BUILD.bazel new file mode 100644 index 00000000..8866f5af --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/api/dto/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_java//java:java_library.bzl", "java_library") + +java_library( + name = "dto", + srcs = [ + "GameFeatureRow.java", + "IndexRequest.java", + "IndexResponse.java", + "QueryRequest.java", + "QueryResponse.java", + ], + visibility = ["//visibility:public"], + deps = [ + "//jvm/src/main/java/com/muchq/indexer/db", + ], +) diff --git a/jvm/src/main/java/com/muchq/indexer/api/dto/GameFeatureRow.java b/jvm/src/main/java/com/muchq/indexer/api/dto/GameFeatureRow.java new file mode 100644 index 00000000..8f019a55 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/api/dto/GameFeatureRow.java @@ -0,0 +1,45 @@ +package com.muchq.indexer.api.dto; + +import com.muchq.indexer.db.GameFeatureStore; + +import java.time.Instant; + +public record GameFeatureRow( + String gameUrl, + String platform, + String whiteUsername, + String blackUsername, + Integer whiteElo, + Integer blackElo, + String timeClass, + String eco, + String result, + Instant playedAt, + Integer numMoves, + boolean hasPin, + boolean hasCrossPin, + boolean hasFork, + boolean hasSkewer, + boolean hasDiscoveredAttack +) { + public static GameFeatureRow fromStore(GameFeatureStore.GameFeature row) { + return new GameFeatureRow( + row.gameUrl(), + row.platform(), + row.whiteUsername(), + row.blackUsername(), + row.whiteElo(), + row.blackElo(), + row.timeClass(), + row.eco(), + row.result(), + row.playedAt(), + row.numMoves(), + row.hasPin(), + row.hasCrossPin(), + row.hasFork(), + row.hasSkewer(), + row.hasDiscoveredAttack() + ); + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/api/dto/IndexRequest.java b/jvm/src/main/java/com/muchq/indexer/api/dto/IndexRequest.java new file mode 100644 index 00000000..ea3321b9 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/api/dto/IndexRequest.java @@ -0,0 +1,4 @@ +package com.muchq.indexer.api.dto; + +public record IndexRequest(String player, String platform, String startMonth, String endMonth) { +} diff --git a/jvm/src/main/java/com/muchq/indexer/api/dto/IndexResponse.java b/jvm/src/main/java/com/muchq/indexer/api/dto/IndexResponse.java new file mode 100644 index 00000000..8a9e9e60 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/api/dto/IndexResponse.java @@ -0,0 +1,6 @@ +package com.muchq.indexer.api.dto; + +import java.util.UUID; + +public record IndexResponse(UUID id, String status, int gamesIndexed, String errorMessage) { +} diff --git a/jvm/src/main/java/com/muchq/indexer/api/dto/QueryRequest.java b/jvm/src/main/java/com/muchq/indexer/api/dto/QueryRequest.java new file mode 100644 index 00000000..45a2724f --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/api/dto/QueryRequest.java @@ -0,0 +1,9 @@ +package com.muchq.indexer.api.dto; + +public record QueryRequest(String query, int limit, int offset) { + public QueryRequest { + if (limit <= 0) limit = 50; + if (limit > 1000) limit = 1000; + if (offset < 0) offset = 0; + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/api/dto/QueryResponse.java b/jvm/src/main/java/com/muchq/indexer/api/dto/QueryResponse.java new file mode 100644 index 00000000..5f9e414f --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/api/dto/QueryResponse.java @@ -0,0 +1,6 @@ +package com.muchq.indexer.api.dto; + +import java.util.List; + +public record QueryResponse(List games, int count) { +} diff --git a/jvm/src/main/java/com/muchq/indexer/chessql/ast/AndExpr.java b/jvm/src/main/java/com/muchq/indexer/chessql/ast/AndExpr.java new file mode 100644 index 00000000..78b45e11 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/chessql/ast/AndExpr.java @@ -0,0 +1,6 @@ +package com.muchq.indexer.chessql.ast; + +import java.util.List; + +public record AndExpr(List operands) implements Expr { +} diff --git a/jvm/src/main/java/com/muchq/indexer/chessql/ast/BUILD.bazel b/jvm/src/main/java/com/muchq/indexer/chessql/ast/BUILD.bazel new file mode 100644 index 00000000..00f87ed3 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/chessql/ast/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_java//java:java_library.bzl", "java_library") + +java_library( + name = "ast", + srcs = [ + "AndExpr.java", + "ComparisonExpr.java", + "Expr.java", + "InExpr.java", + "MotifExpr.java", + "NotExpr.java", + "OrExpr.java", + ], + visibility = ["//visibility:public"], +) diff --git a/jvm/src/main/java/com/muchq/indexer/chessql/ast/ComparisonExpr.java b/jvm/src/main/java/com/muchq/indexer/chessql/ast/ComparisonExpr.java new file mode 100644 index 00000000..f568b0e5 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/chessql/ast/ComparisonExpr.java @@ -0,0 +1,4 @@ +package com.muchq.indexer.chessql.ast; + +public record ComparisonExpr(String field, String operator, Object value) implements Expr { +} diff --git a/jvm/src/main/java/com/muchq/indexer/chessql/ast/Expr.java b/jvm/src/main/java/com/muchq/indexer/chessql/ast/Expr.java new file mode 100644 index 00000000..1726960c --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/chessql/ast/Expr.java @@ -0,0 +1,4 @@ +package com.muchq.indexer.chessql.ast; + +public sealed interface Expr permits OrExpr, AndExpr, NotExpr, ComparisonExpr, InExpr, MotifExpr { +} diff --git a/jvm/src/main/java/com/muchq/indexer/chessql/ast/InExpr.java b/jvm/src/main/java/com/muchq/indexer/chessql/ast/InExpr.java new file mode 100644 index 00000000..6dcf94a7 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/chessql/ast/InExpr.java @@ -0,0 +1,6 @@ +package com.muchq.indexer.chessql.ast; + +import java.util.List; + +public record InExpr(String field, List values) implements Expr { +} diff --git a/jvm/src/main/java/com/muchq/indexer/chessql/ast/MotifExpr.java b/jvm/src/main/java/com/muchq/indexer/chessql/ast/MotifExpr.java new file mode 100644 index 00000000..8c482ef6 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/chessql/ast/MotifExpr.java @@ -0,0 +1,4 @@ +package com.muchq.indexer.chessql.ast; + +public record MotifExpr(String motifName) implements Expr { +} diff --git a/jvm/src/main/java/com/muchq/indexer/chessql/ast/NotExpr.java b/jvm/src/main/java/com/muchq/indexer/chessql/ast/NotExpr.java new file mode 100644 index 00000000..06e5fcbc --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/chessql/ast/NotExpr.java @@ -0,0 +1,4 @@ +package com.muchq.indexer.chessql.ast; + +public record NotExpr(Expr operand) implements Expr { +} diff --git a/jvm/src/main/java/com/muchq/indexer/chessql/ast/OrExpr.java b/jvm/src/main/java/com/muchq/indexer/chessql/ast/OrExpr.java new file mode 100644 index 00000000..2e521f8d --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/chessql/ast/OrExpr.java @@ -0,0 +1,6 @@ +package com.muchq.indexer.chessql.ast; + +import java.util.List; + +public record OrExpr(List operands) implements Expr { +} diff --git a/jvm/src/main/java/com/muchq/indexer/chessql/compiler/BUILD.bazel b/jvm/src/main/java/com/muchq/indexer/chessql/compiler/BUILD.bazel new file mode 100644 index 00000000..ec98ed86 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/chessql/compiler/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_java//java:java_library.bzl", "java_library") + +java_library( + name = "compiler", + srcs = [ + "CompiledQuery.java", + "QueryCompiler.java", + "SqlCompiler.java", + ], + visibility = ["//visibility:public"], + deps = [ + "//jvm/src/main/java/com/muchq/indexer/chessql/ast", + ], +) diff --git a/jvm/src/main/java/com/muchq/indexer/chessql/compiler/CompiledQuery.java b/jvm/src/main/java/com/muchq/indexer/chessql/compiler/CompiledQuery.java new file mode 100644 index 00000000..d9a58376 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/chessql/compiler/CompiledQuery.java @@ -0,0 +1,6 @@ +package com.muchq.indexer.chessql.compiler; + +import java.util.List; + +public record CompiledQuery(String sql, List parameters) { +} diff --git a/jvm/src/main/java/com/muchq/indexer/chessql/compiler/QueryCompiler.java b/jvm/src/main/java/com/muchq/indexer/chessql/compiler/QueryCompiler.java new file mode 100644 index 00000000..41880569 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/chessql/compiler/QueryCompiler.java @@ -0,0 +1,7 @@ +package com.muchq.indexer.chessql.compiler; + +import com.muchq.indexer.chessql.ast.Expr; + +public interface QueryCompiler { + T compile(Expr expr); +} diff --git a/jvm/src/main/java/com/muchq/indexer/chessql/compiler/SqlCompiler.java b/jvm/src/main/java/com/muchq/indexer/chessql/compiler/SqlCompiler.java new file mode 100644 index 00000000..5d1aac6a --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/chessql/compiler/SqlCompiler.java @@ -0,0 +1,105 @@ +package com.muchq.indexer.chessql.compiler; + +import com.muchq.indexer.chessql.ast.AndExpr; +import com.muchq.indexer.chessql.ast.ComparisonExpr; +import com.muchq.indexer.chessql.ast.Expr; +import com.muchq.indexer.chessql.ast.InExpr; +import com.muchq.indexer.chessql.ast.MotifExpr; +import com.muchq.indexer.chessql.ast.NotExpr; +import com.muchq.indexer.chessql.ast.OrExpr; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class SqlCompiler implements QueryCompiler { + private static final Set VALID_COLUMNS = Set.of( + "white_username", "black_username", "white_elo", "black_elo", + "time_class", "eco", "result", "num_moves", "platform", + "game_url", "played_at" + ); + + private static final Set VALID_MOTIFS = Set.of( + "pin", "cross_pin", "fork", "skewer", "discovered_attack" + ); + + private static final Map FIELD_MAP = Map.of( + "white.elo", "white_elo", + "black.elo", "black_elo", + "white.username", "white_username", + "black.username", "black_username", + "time.class", "time_class", + "num.moves", "num_moves", + "game.url", "game_url", + "played.at", "played_at" + ); + + private static final Set VALID_OPS = Set.of("=", "!=", "<", "<=", ">", ">="); + + public CompiledQuery compile(Expr expr) { + List params = new ArrayList<>(); + String sql = compileExpr(expr, params); + return new CompiledQuery(sql, params); + } + + private String compileExpr(Expr expr, List params) { + return switch (expr) { + case OrExpr or -> or.operands().stream() + .map(e -> compileExpr(e, params)) + .collect(Collectors.joining(" OR ", "(", ")")); + case AndExpr and -> and.operands().stream() + .map(e -> compileExpr(e, params)) + .collect(Collectors.joining(" AND ", "(", ")")); + case NotExpr not -> "(NOT " + compileExpr(not.operand(), params) + ")"; + case ComparisonExpr cmp -> compileComparison(cmp, params); + case InExpr in -> compileIn(in, params); + case MotifExpr motif -> compileMotif(motif); + }; + } + + private String compileComparison(ComparisonExpr cmp, List params) { + String column = resolveColumn(cmp.field()); + String op = cmp.operator(); + if (!VALID_OPS.contains(op)) { + throw new IllegalArgumentException("Invalid operator: " + op); + } + params.add(cmp.value()); + return column + " " + op + " ?"; + } + + private String compileIn(InExpr in, List params) { + String column = resolveColumn(in.field()); + params.addAll(in.values()); + String placeholders = in.values().stream() + .map(v -> "?") + .collect(Collectors.joining(", ")); + return column + " IN (" + placeholders + ")"; + } + + private String compileMotif(MotifExpr motif) { + String name = motif.motifName(); + if (!VALID_MOTIFS.contains(name)) { + throw new IllegalArgumentException("Unknown motif: " + name); + } + return "has_" + name + " = TRUE"; + } + + private String resolveColumn(String field) { + String mapped = FIELD_MAP.get(field); + if (mapped != null) { + return mapped; + } + // Try direct column name (already underscore-separated) + if (VALID_COLUMNS.contains(field)) { + return field; + } + // Try converting dots to underscores + String underscored = field.replace('.', '_'); + if (VALID_COLUMNS.contains(underscored)) { + return underscored; + } + throw new IllegalArgumentException("Unknown field: " + field); + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/chessql/lexer/BUILD.bazel b/jvm/src/main/java/com/muchq/indexer/chessql/lexer/BUILD.bazel new file mode 100644 index 00000000..b8ccef89 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/chessql/lexer/BUILD.bazel @@ -0,0 +1,11 @@ +load("@rules_java//java:java_library.bzl", "java_library") + +java_library( + name = "lexer", + srcs = [ + "Lexer.java", + "Token.java", + "TokenType.java", + ], + visibility = ["//visibility:public"], +) diff --git a/jvm/src/main/java/com/muchq/indexer/chessql/lexer/Lexer.java b/jvm/src/main/java/com/muchq/indexer/chessql/lexer/Lexer.java new file mode 100644 index 00000000..41c6e7c8 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/chessql/lexer/Lexer.java @@ -0,0 +1,120 @@ +package com.muchq.indexer.chessql.lexer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class Lexer { + private static final Map KEYWORDS = Map.of( + "AND", TokenType.AND, + "OR", TokenType.OR, + "NOT", TokenType.NOT, + "IN", TokenType.IN, + "motif", TokenType.MOTIF + ); + + private final String input; + private int pos; + + public Lexer(String input) { + this.input = input; + this.pos = 0; + } + + public List tokenize() { + List tokens = new ArrayList<>(); + while (pos < input.length()) { + char c = input.charAt(pos); + + if (Character.isWhitespace(c)) { + pos++; + continue; + } + + if (c == '(') { + tokens.add(new Token(TokenType.LPAREN, "(", pos++)); + } else if (c == ')') { + tokens.add(new Token(TokenType.RPAREN, ")", pos++)); + } else if (c == '[') { + tokens.add(new Token(TokenType.LBRACKET, "[", pos++)); + } else if (c == ']') { + tokens.add(new Token(TokenType.RBRACKET, "]", pos++)); + } else if (c == ',') { + tokens.add(new Token(TokenType.COMMA, ",", pos++)); + } else if (c == '.') { + tokens.add(new Token(TokenType.DOT, ".", pos++)); + } else if (c == '=') { + tokens.add(new Token(TokenType.EQ, "=", pos++)); + } else if (c == '!' && peek() == '=') { + tokens.add(new Token(TokenType.NEQ, "!=", pos)); + pos += 2; + } else if (c == '<' && peek() == '=') { + tokens.add(new Token(TokenType.LTE, "<=", pos)); + pos += 2; + } else if (c == '<') { + tokens.add(new Token(TokenType.LT, "<", pos++)); + } else if (c == '>' && peek() == '=') { + tokens.add(new Token(TokenType.GTE, ">=", pos)); + pos += 2; + } else if (c == '>') { + tokens.add(new Token(TokenType.GT, ">", pos++)); + } else if (c == '"') { + tokens.add(readString()); + } else if (Character.isDigit(c) || (c == '-' && pos + 1 < input.length() && Character.isDigit(input.charAt(pos + 1)))) { + tokens.add(readNumber()); + } else if (Character.isLetter(c) || c == '_') { + tokens.add(readIdentifierOrKeyword()); + } else { + throw new IllegalArgumentException("Unexpected character '" + c + "' at position " + pos); + } + } + + tokens.add(new Token(TokenType.EOF, "", pos)); + return tokens; + } + + private char peek() { + return pos + 1 < input.length() ? input.charAt(pos + 1) : '\0'; + } + + private Token readString() { + int start = pos; + pos++; // skip opening quote + StringBuilder sb = new StringBuilder(); + while (pos < input.length() && input.charAt(pos) != '"') { + if (input.charAt(pos) == '\\' && pos + 1 < input.length()) { + pos++; + sb.append(input.charAt(pos)); + } else { + sb.append(input.charAt(pos)); + } + pos++; + } + if (pos >= input.length()) { + throw new IllegalArgumentException("Unterminated string at position " + start); + } + pos++; // skip closing quote + return new Token(TokenType.STRING, sb.toString(), start); + } + + private Token readNumber() { + int start = pos; + if (input.charAt(pos) == '-') { + pos++; + } + while (pos < input.length() && Character.isDigit(input.charAt(pos))) { + pos++; + } + return new Token(TokenType.NUMBER, input.substring(start, pos), start); + } + + private Token readIdentifierOrKeyword() { + int start = pos; + while (pos < input.length() && (Character.isLetterOrDigit(input.charAt(pos)) || input.charAt(pos) == '_')) { + pos++; + } + String word = input.substring(start, pos); + TokenType type = KEYWORDS.getOrDefault(word, TokenType.IDENTIFIER); + return new Token(type, word, start); + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/chessql/lexer/Token.java b/jvm/src/main/java/com/muchq/indexer/chessql/lexer/Token.java new file mode 100644 index 00000000..68322096 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/chessql/lexer/Token.java @@ -0,0 +1,8 @@ +package com.muchq.indexer.chessql.lexer; + +public record Token(TokenType type, String value, int position) { + @Override + public String toString() { + return "Token(" + type + ", " + value + ", pos=" + position + ")"; + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/chessql/lexer/TokenType.java b/jvm/src/main/java/com/muchq/indexer/chessql/lexer/TokenType.java new file mode 100644 index 00000000..816776fa --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/chessql/lexer/TokenType.java @@ -0,0 +1,34 @@ +package com.muchq.indexer.chessql.lexer; + +public enum TokenType { + // Literals + NUMBER, + STRING, + IDENTIFIER, + + // Operators + EQ, // = + NEQ, // != + LT, // < + LTE, // <= + GT, // > + GTE, // >= + + // Keywords + AND, + OR, + NOT, + IN, + MOTIF, + + // Delimiters + LPAREN, + RPAREN, + LBRACKET, + RBRACKET, + COMMA, + DOT, + + // End + EOF +} diff --git a/jvm/src/main/java/com/muchq/indexer/chessql/parser/BUILD.bazel b/jvm/src/main/java/com/muchq/indexer/chessql/parser/BUILD.bazel new file mode 100644 index 00000000..18c54e20 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/chessql/parser/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_java//java:java_library.bzl", "java_library") + +java_library( + name = "parser", + srcs = [ + "ParseException.java", + "Parser.java", + ], + visibility = ["//visibility:public"], + deps = [ + "//jvm/src/main/java/com/muchq/indexer/chessql/ast", + "//jvm/src/main/java/com/muchq/indexer/chessql/lexer", + ], +) diff --git a/jvm/src/main/java/com/muchq/indexer/chessql/parser/ParseException.java b/jvm/src/main/java/com/muchq/indexer/chessql/parser/ParseException.java new file mode 100644 index 00000000..0b3b0dfc --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/chessql/parser/ParseException.java @@ -0,0 +1,14 @@ +package com.muchq.indexer.chessql.parser; + +public class ParseException extends RuntimeException { + private final int position; + + public ParseException(String message, int position) { + super(message + " at position " + position); + this.position = position; + } + + public int getPosition() { + return position; + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/chessql/parser/Parser.java b/jvm/src/main/java/com/muchq/indexer/chessql/parser/Parser.java new file mode 100644 index 00000000..7673894c --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/chessql/parser/Parser.java @@ -0,0 +1,184 @@ +package com.muchq.indexer.chessql.parser; + +import com.muchq.indexer.chessql.ast.AndExpr; +import com.muchq.indexer.chessql.ast.ComparisonExpr; +import com.muchq.indexer.chessql.ast.Expr; +import com.muchq.indexer.chessql.ast.InExpr; +import com.muchq.indexer.chessql.ast.MotifExpr; +import com.muchq.indexer.chessql.ast.NotExpr; +import com.muchq.indexer.chessql.ast.OrExpr; +import com.muchq.indexer.chessql.lexer.Lexer; +import com.muchq.indexer.chessql.lexer.Token; +import com.muchq.indexer.chessql.lexer.TokenType; + +import java.util.ArrayList; +import java.util.List; + +public class Parser { + private final List tokens; + private int pos; + + public Parser(List tokens) { + this.tokens = tokens; + this.pos = 0; + } + + public static Expr parse(String input) { + List tokens = new Lexer(input).tokenize(); + Parser parser = new Parser(tokens); + Expr expr = parser.parseExpr(); + parser.expect(TokenType.EOF); + return expr; + } + + public Expr parseExpr() { + return parseOr(); + } + + private Expr parseOr() { + Expr left = parseAnd(); + List operands = new ArrayList<>(); + operands.add(left); + + while (check(TokenType.OR)) { + advance(); + operands.add(parseAnd()); + } + + return operands.size() == 1 ? operands.get(0) : new OrExpr(operands); + } + + private Expr parseAnd() { + Expr left = parseNot(); + List operands = new ArrayList<>(); + operands.add(left); + + while (check(TokenType.AND)) { + advance(); + operands.add(parseNot()); + } + + return operands.size() == 1 ? operands.get(0) : new AndExpr(operands); + } + + private Expr parseNot() { + if (check(TokenType.NOT)) { + advance(); + return new NotExpr(parseNot()); + } + return parsePrimary(); + } + + private Expr parsePrimary() { + if (check(TokenType.LPAREN)) { + advance(); + Expr expr = parseExpr(); + expect(TokenType.RPAREN); + return expr; + } + + if (check(TokenType.MOTIF)) { + return parseMotif(); + } + + if (check(TokenType.IDENTIFIER)) { + return parseFieldExpr(); + } + + throw new ParseException("Unexpected token: " + current(), current().position()); + } + + private Expr parseMotif() { + advance(); // consume 'motif' + expect(TokenType.LPAREN); + Token name = expect(TokenType.IDENTIFIER); + expect(TokenType.RPAREN); + return new MotifExpr(name.value()); + } + + private Expr parseFieldExpr() { + String field = parseFieldName(); + + if (check(TokenType.IN)) { + advance(); + return parseInValues(field); + } + + String op = parseCompOp(); + Object value = parseValue(); + return new ComparisonExpr(field, op, value); + } + + private String parseFieldName() { + Token first = expect(TokenType.IDENTIFIER); + StringBuilder sb = new StringBuilder(first.value()); + + while (check(TokenType.DOT)) { + advance(); + Token next = expect(TokenType.IDENTIFIER); + sb.append('.').append(next.value()); + } + + return sb.toString(); + } + + private String parseCompOp() { + Token t = current(); + return switch (t.type()) { + case EQ -> { advance(); yield "="; } + case NEQ -> { advance(); yield "!="; } + case LT -> { advance(); yield "<"; } + case LTE -> { advance(); yield "<="; } + case GT -> { advance(); yield ">"; } + case GTE -> { advance(); yield ">="; } + default -> throw new ParseException("Expected comparison operator, got: " + t, t.position()); + }; + } + + private Object parseValue() { + Token t = current(); + if (t.type() == TokenType.NUMBER) { + advance(); + return Integer.parseInt(t.value()); + } + if (t.type() == TokenType.STRING) { + advance(); + return t.value(); + } + throw new ParseException("Expected value, got: " + t, t.position()); + } + + private InExpr parseInValues(String field) { + expect(TokenType.LBRACKET); + List values = new ArrayList<>(); + values.add(parseValue()); + while (check(TokenType.COMMA)) { + advance(); + values.add(parseValue()); + } + expect(TokenType.RBRACKET); + return new InExpr(field, values); + } + + private Token current() { + return tokens.get(pos); + } + + private boolean check(TokenType type) { + return current().type() == type; + } + + private Token advance() { + Token t = tokens.get(pos); + pos++; + return t; + } + + private Token expect(TokenType type) { + Token t = current(); + if (t.type() != type) { + throw new ParseException("Expected " + type + ", got " + t.type(), t.position()); + } + return advance(); + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/db/BUILD.bazel b/jvm/src/main/java/com/muchq/indexer/db/BUILD.bazel new file mode 100644 index 00000000..52d84ee2 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/db/BUILD.bazel @@ -0,0 +1,20 @@ +load("@rules_java//java:java_library.bzl", "java_library") + +java_library( + name = "db", + srcs = [ + "DataSourceFactory.java", + "GameFeatureDao.java", + "GameFeatureStore.java", + "IndexingRequestDao.java", + "IndexingRequestStore.java", + "Migration.java", + ], + visibility = ["//visibility:public"], + deps = [ + "//jvm/src/main/java/com/muchq/indexer/chessql/compiler", + "@maven//:com_h2database_h2", + "@maven//:com_zaxxer_HikariCP", + "@maven//:org_slf4j_slf4j_api", + ], +) diff --git a/jvm/src/main/java/com/muchq/indexer/db/DataSourceFactory.java b/jvm/src/main/java/com/muchq/indexer/db/DataSourceFactory.java new file mode 100644 index 00000000..7e758ac5 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/db/DataSourceFactory.java @@ -0,0 +1,20 @@ +package com.muchq.indexer.db; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; + +import javax.sql.DataSource; + +public class DataSourceFactory { + private DataSourceFactory() {} + + public static DataSource create(String jdbcUrl, String username, String password) { + HikariConfig config = new HikariConfig(); + config.setJdbcUrl(jdbcUrl); + config.setUsername(username); + config.setPassword(password); + config.setMaximumPoolSize(10); + config.setMinimumIdle(2); + return new HikariDataSource(config); + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/db/GameFeatureDao.java b/jvm/src/main/java/com/muchq/indexer/db/GameFeatureDao.java new file mode 100644 index 00000000..ddde060b --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/db/GameFeatureDao.java @@ -0,0 +1,144 @@ +package com.muchq.indexer.db; + +import com.muchq.indexer.chessql.compiler.CompiledQuery; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +public class GameFeatureDao implements GameFeatureStore { + private static final Logger LOG = LoggerFactory.getLogger(GameFeatureDao.class); + + private static final String H2_INSERT = """ + MERGE INTO game_features ( + request_id, game_url, platform, white_username, black_username, + white_elo, black_elo, time_class, eco, result, played_at, num_moves, + has_pin, has_cross_pin, has_fork, has_skewer, has_discovered_attack, + motifs_json, pgn + ) KEY (game_url) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """; + + private static final String PG_INSERT = """ + INSERT INTO game_features ( + request_id, game_url, platform, white_username, black_username, + white_elo, black_elo, time_class, eco, result, played_at, num_moves, + has_pin, has_cross_pin, has_fork, has_skewer, has_discovered_attack, + motifs_json, pgn + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb, ?) + ON CONFLICT (game_url) DO NOTHING + """; + + private final DataSource dataSource; + private final boolean useH2; + + public GameFeatureDao(DataSource dataSource, boolean useH2) { + this.dataSource = dataSource; + this.useH2 = useH2; + } + + @Override + public void insert(GameFeature row) { + String sql = useH2 ? H2_INSERT : PG_INSERT; + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setObject(1, row.requestId()); + ps.setString(2, row.gameUrl()); + ps.setString(3, row.platform()); + ps.setString(4, row.whiteUsername()); + ps.setString(5, row.blackUsername()); + setIntOrNull(ps, 6, row.whiteElo()); + setIntOrNull(ps, 7, row.blackElo()); + ps.setString(8, row.timeClass()); + ps.setString(9, row.eco()); + ps.setString(10, row.result()); + ps.setTimestamp(11, row.playedAt() != null ? Timestamp.from(row.playedAt()) : null); + setIntOrNull(ps, 12, row.numMoves()); + ps.setBoolean(13, row.hasPin()); + ps.setBoolean(14, row.hasCrossPin()); + ps.setBoolean(15, row.hasFork()); + ps.setBoolean(16, row.hasSkewer()); + ps.setBoolean(17, row.hasDiscoveredAttack()); + ps.setString(18, row.motifsJson()); + ps.setString(19, row.pgn()); + ps.executeUpdate(); + } catch (SQLException e) { + LOG.error("Failed to insert game feature for game_url={}", row.gameUrl(), e); + throw new RuntimeException("Failed to insert game feature", e); + } + } + + @Override + public List query(Object compiledQuery, int limit, int offset) { + if (!(compiledQuery instanceof CompiledQuery cq)) { + throw new IllegalArgumentException("Expected CompiledQuery, got: " + compiledQuery.getClass()); + } + String sql = "SELECT * FROM game_features WHERE " + cq.sql() + + " LIMIT ? OFFSET ?"; + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + int idx = 1; + for (Object param : cq.parameters()) { + ps.setObject(idx++, param); + } + ps.setInt(idx++, limit); + ps.setInt(idx, offset); + + try (ResultSet rs = ps.executeQuery()) { + List results = new ArrayList<>(); + while (rs.next()) { + results.add(mapRow(rs)); + } + return results; + } + } catch (SQLException e) { + throw new RuntimeException("Failed to query game features", e); + } + } + + private GameFeature mapRow(ResultSet rs) throws SQLException { + return new GameFeature( + UUID.fromString(rs.getString("id")), + UUID.fromString(rs.getString("request_id")), + rs.getString("game_url"), + rs.getString("platform"), + rs.getString("white_username"), + rs.getString("black_username"), + getIntOrNull(rs, "white_elo"), + getIntOrNull(rs, "black_elo"), + rs.getString("time_class"), + rs.getString("eco"), + rs.getString("result"), + rs.getTimestamp("played_at") != null ? rs.getTimestamp("played_at").toInstant() : null, + getIntOrNull(rs, "num_moves"), + rs.getBoolean("has_pin"), + rs.getBoolean("has_cross_pin"), + rs.getBoolean("has_fork"), + rs.getBoolean("has_skewer"), + rs.getBoolean("has_discovered_attack"), + rs.getString("motifs_json"), + rs.getString("pgn") + ); + } + + private static void setIntOrNull(PreparedStatement ps, int idx, Integer value) throws SQLException { + if (value != null) { + ps.setInt(idx, value); + } else { + ps.setNull(idx, Types.INTEGER); + } + } + + private static Integer getIntOrNull(ResultSet rs, String column) throws SQLException { + int val = rs.getInt(column); + return rs.wasNull() ? null : val; + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/db/GameFeatureStore.java b/jvm/src/main/java/com/muchq/indexer/db/GameFeatureStore.java new file mode 100644 index 00000000..3abd9768 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/db/GameFeatureStore.java @@ -0,0 +1,33 @@ +package com.muchq.indexer.db; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +public interface GameFeatureStore { + void insert(GameFeature feature); + List query(Object compiledQuery, int limit, int offset); + + record GameFeature( + UUID id, + UUID requestId, + String gameUrl, + String platform, + String whiteUsername, + String blackUsername, + Integer whiteElo, + Integer blackElo, + String timeClass, + String eco, + String result, + Instant playedAt, + Integer numMoves, + boolean hasPin, + boolean hasCrossPin, + boolean hasFork, + boolean hasSkewer, + boolean hasDiscoveredAttack, + String motifsJson, + String pgn + ) {} +} diff --git a/jvm/src/main/java/com/muchq/indexer/db/IndexingRequestDao.java b/jvm/src/main/java/com/muchq/indexer/db/IndexingRequestDao.java new file mode 100644 index 00000000..aefa30ce --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/db/IndexingRequestDao.java @@ -0,0 +1,94 @@ +package com.muchq.indexer.db; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Optional; +import java.util.UUID; + +public class IndexingRequestDao implements IndexingRequestStore { + private static final Logger LOG = LoggerFactory.getLogger(IndexingRequestDao.class); + + private final DataSource dataSource; + + public IndexingRequestDao(DataSource dataSource) { + this.dataSource = dataSource; + } + + @Override + public UUID create(String player, String platform, String startMonth, String endMonth) { + UUID id = UUID.randomUUID(); + String sql = """ + INSERT INTO indexing_requests (id, player, platform, start_month, end_month) + VALUES (?, ?, ?, ?, ?) + """; + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setObject(1, id); + ps.setString(2, player); + ps.setString(3, platform); + ps.setString(4, startMonth); + ps.setString(5, endMonth); + ps.executeUpdate(); + return id; + } catch (SQLException e) { + throw new RuntimeException("Failed to create indexing request", e); + } + } + + @Override + public Optional findById(UUID id) { + String sql = "SELECT * FROM indexing_requests WHERE id = ?"; + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setObject(1, id); + try (ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + return Optional.of(mapRow(rs)); + } + return Optional.empty(); + } + } catch (SQLException e) { + throw new RuntimeException("Failed to find indexing request", e); + } + } + + @Override + public void updateStatus(UUID id, String status, String errorMessage, int gamesIndexed) { + String sql = """ + UPDATE indexing_requests + SET status = ?, error_message = ?, games_indexed = ?, updated_at = now() + WHERE id = ? + """; + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setString(1, status); + ps.setString(2, errorMessage); + ps.setInt(3, gamesIndexed); + ps.setObject(4, id); + ps.executeUpdate(); + } catch (SQLException e) { + throw new RuntimeException("Failed to update indexing request status", e); + } + } + + private IndexingRequest mapRow(ResultSet rs) throws SQLException { + return new IndexingRequest( + UUID.fromString(rs.getString("id")), + rs.getString("player"), + rs.getString("platform"), + rs.getString("start_month"), + rs.getString("end_month"), + rs.getString("status"), + rs.getTimestamp("created_at").toInstant(), + rs.getTimestamp("updated_at").toInstant(), + rs.getString("error_message"), + rs.getInt("games_indexed") + ); + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/db/IndexingRequestStore.java b/jvm/src/main/java/com/muchq/indexer/db/IndexingRequestStore.java new file mode 100644 index 00000000..40af1a46 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/db/IndexingRequestStore.java @@ -0,0 +1,24 @@ +package com.muchq.indexer.db; + +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; + +public interface IndexingRequestStore { + UUID create(String player, String platform, String startMonth, String endMonth); + Optional findById(UUID id); + void updateStatus(UUID id, String status, String errorMessage, int gamesIndexed); + + record IndexingRequest( + UUID id, + String player, + String platform, + String startMonth, + String endMonth, + String status, + Instant createdAt, + Instant updatedAt, + String errorMessage, + int gamesIndexed + ) {} +} diff --git a/jvm/src/main/java/com/muchq/indexer/db/Migration.java b/jvm/src/main/java/com/muchq/indexer/db/Migration.java new file mode 100644 index 00000000..0ac3a8d0 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/db/Migration.java @@ -0,0 +1,119 @@ +package com.muchq.indexer.db; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + +public class Migration { + private static final Logger LOG = LoggerFactory.getLogger(Migration.class); + + private static final String H2_INDEXING_REQUESTS = """ + CREATE TABLE IF NOT EXISTS indexing_requests ( + id UUID DEFAULT random_uuid() PRIMARY KEY, + player VARCHAR(255) NOT NULL, + platform VARCHAR(50) NOT NULL, + start_month VARCHAR(7) NOT NULL, + end_month VARCHAR(7) NOT NULL, + status VARCHAR(20) NOT NULL DEFAULT 'PENDING', + created_at TIMESTAMP NOT NULL DEFAULT current_timestamp(), + updated_at TIMESTAMP NOT NULL DEFAULT current_timestamp(), + error_message TEXT, + games_indexed INT DEFAULT 0 + ) + """; + + private static final String H2_GAME_FEATURES = """ + CREATE TABLE IF NOT EXISTS game_features ( + id UUID DEFAULT random_uuid() PRIMARY KEY, + request_id UUID NOT NULL REFERENCES indexing_requests(id), + game_url VARCHAR(1024) NOT NULL UNIQUE, + platform VARCHAR(50) NOT NULL, + white_username VARCHAR(255), + black_username VARCHAR(255), + white_elo INT, + black_elo INT, + time_class VARCHAR(50), + eco VARCHAR(10), + result VARCHAR(20), + played_at TIMESTAMP, + num_moves INT, + has_pin BOOLEAN DEFAULT FALSE, + has_cross_pin BOOLEAN DEFAULT FALSE, + has_fork BOOLEAN DEFAULT FALSE, + has_skewer BOOLEAN DEFAULT FALSE, + has_discovered_attack BOOLEAN DEFAULT FALSE, + motifs_json TEXT, + pgn TEXT + ) + """; + + private static final String PG_INDEXING_REQUESTS = """ + CREATE TABLE IF NOT EXISTS indexing_requests ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + player VARCHAR(255) NOT NULL, + platform VARCHAR(50) NOT NULL, + start_month VARCHAR(7) NOT NULL, + end_month VARCHAR(7) NOT NULL, + status VARCHAR(20) NOT NULL DEFAULT 'PENDING', + created_at TIMESTAMP NOT NULL DEFAULT now(), + updated_at TIMESTAMP NOT NULL DEFAULT now(), + error_message TEXT, + games_indexed INT DEFAULT 0 + ) + """; + + private static final String PG_GAME_FEATURES = """ + CREATE TABLE IF NOT EXISTS game_features ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + request_id UUID NOT NULL REFERENCES indexing_requests(id), + game_url VARCHAR(1024) NOT NULL UNIQUE, + platform VARCHAR(50) NOT NULL, + white_username VARCHAR(255), + black_username VARCHAR(255), + white_elo INT, + black_elo INT, + time_class VARCHAR(50), + eco VARCHAR(10), + result VARCHAR(20), + played_at TIMESTAMP, + num_moves INT, + has_pin BOOLEAN DEFAULT FALSE, + has_cross_pin BOOLEAN DEFAULT FALSE, + has_fork BOOLEAN DEFAULT FALSE, + has_skewer BOOLEAN DEFAULT FALSE, + has_discovered_attack BOOLEAN DEFAULT FALSE, + motifs_json JSONB, + pgn TEXT + ) + """; + + private final DataSource dataSource; + private final boolean useH2; + + public Migration(DataSource dataSource, boolean useH2) { + this.dataSource = dataSource; + this.useH2 = useH2; + } + + public void run() { + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement()) { + + if (useH2) { + stmt.execute(H2_INDEXING_REQUESTS); + stmt.execute(H2_GAME_FEATURES); + } else { + stmt.execute(PG_INDEXING_REQUESTS); + stmt.execute(PG_GAME_FEATURES); + } + + LOG.info("Database migration completed successfully (H2={})", useH2); + } catch (SQLException e) { + throw new RuntimeException("Failed to run database migration", e); + } + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/docs/API.md b/jvm/src/main/java/com/muchq/indexer/docs/API.md new file mode 100644 index 00000000..55877c99 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/docs/API.md @@ -0,0 +1,175 @@ +# Chess Game Indexer — API Reference + +## Base URL + +``` +http://localhost:8080 +``` + +Configurable via `PORT` environment variable. + +--- + +## POST /index + +Start indexing games for a player over a month range. + +### Request + +```json +{ + "player": "hikaru", + "platform": "CHESS_COM", + "startMonth": "2024-03", + "endMonth": "2024-03" +} +``` + +| Field | Type | Required | Description | +|-------------|--------|----------|----------------------------------------| +| player | string | yes | Username on the chess platform | +| platform | string | yes | `"CHESS_COM"` (lichess planned) | +| startMonth | string | yes | Start month inclusive, format `YYYY-MM` | +| endMonth | string | yes | End month inclusive, format `YYYY-MM` | + +### Response (201) + +```json +{ + "id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "status": "PENDING", + "gamesIndexed": 0, + "errorMessage": null +} +``` + +### Status Lifecycle + +``` +PENDING → PROCESSING → COMPLETED + → FAILED (with errorMessage) +``` + +--- + +## GET /index/{id} + +Poll the status of an indexing request. + +### Response (200) + +```json +{ + "id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "status": "COMPLETED", + "gamesIndexed": 147, + "errorMessage": null +} +``` + +### Response (404 — via RuntimeException, needs error mapping) + +Returned when the ID does not match any indexing request. + +--- + +## POST /query + +Search indexed games using ChessQL. + +### Request + +```json +{ + "query": "white.elo >= 2500 AND motif(fork)", + "limit": 10, + "offset": 0 +} +``` + +| Field | Type | Required | Default | Max | Description | +|--------|--------|----------|---------|------|---------------------------------| +| query | string | yes | — | — | ChessQL query string | +| limit | int | no | 50 | 1000 | Max results to return | +| offset | int | no | 0 | — | Pagination offset | + +### Response (200) + +```json +{ + "games": [ + { + "gameUrl": "https://www.chess.com/game/live/12345", + "platform": "CHESS_COM", + "whiteUsername": "Hikaru", + "blackUsername": "MagnusCarlsen", + "whiteElo": 2850, + "blackElo": 2830, + "timeClass": "blitz", + "eco": "B90", + "result": "1-0", + "playedAt": 1710524400.0, + "numMoves": 42, + "hasPin": true, + "hasCrossPin": false, + "hasFork": true, + "hasSkewer": false, + "hasDiscoveredAttack": false + } + ], + "count": 1 +} +``` + +**Result values:** +- `1-0` — White wins +- `0-1` — Black wins +- `1/2-1/2` — Draw (stalemate, repetition, agreement, etc.) +- `unknown` — Result could not be determined +``` + +### Error Responses + +| Condition | HTTP Status | Cause | +|--------------------|-------------|--------------------------------------| +| Bad ChessQL syntax | 500 | `ParseException` (needs error mapping) | +| Unknown field | 500 | `IllegalArgumentException` | +| Unknown motif | 500 | `IllegalArgumentException` | + +> **Note**: Error mapping to proper 400 responses is a planned improvement. See ROADMAP.md Phase 2. + +--- + +## Example Session + +```bash +# Start the service (in-process mode with H2) +INDEXER_DB_URL="jdbc:h2:mem:indexer;DB_CLOSE_DELAY=-1" bazel run //jvm/src/main/java/com/muchq/indexer:indexer + +# 1. Start indexing +curl -X POST http://localhost:8080/index \ + -H "Content-Type: application/json" \ + -d '{"player":"hikaru","platform":"CHESS_COM","startMonth":"2026-01","endMonth":"2026-01"}' + +# Response: {"id":"abc-123","status":"PENDING","gamesIndexed":0} + +# 2. Poll until completed +curl http://localhost:8080/index/abc-123 + +# Response: {"id":"abc-123","status":"COMPLETED","gamesIndexed":828} + +# 3. Query indexed games +curl -X POST http://localhost:8080/query \ + -H "Content-Type: application/json" \ + -d '{"query":"white.elo > 2700 AND motif(fork)","limit":10,"offset":0}' + +# 4. Query games with multiple motifs +curl -X POST http://localhost:8080/query \ + -H "Content-Type: application/json" \ + -d '{"query":"motif(pin) AND motif(skewer)","limit":10,"offset":0}' + +# 5. Query by ECO opening code +curl -X POST http://localhost:8080/query \ + -H "Content-Type: application/json" \ + -d '{"query":"eco = \"B90\"","limit":10,"offset":0}' +``` diff --git a/jvm/src/main/java/com/muchq/indexer/docs/CHESSQL.md b/jvm/src/main/java/com/muchq/indexer/docs/CHESSQL.md new file mode 100644 index 00000000..8ce8efdf --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/docs/CHESSQL.md @@ -0,0 +1,136 @@ +# ChessQL — Query Language Reference + +## Overview + +ChessQL is a domain-specific query language for searching indexed chess games. Queries are compiled to parameterized SQL at runtime — no string interpolation, no injection risk. + +## Grammar (EBNF) + +``` +query ::= expr EOF +expr ::= or_expr +or_expr ::= and_expr ("OR" and_expr)* +and_expr ::= not_expr ("AND" not_expr)* +not_expr ::= "NOT" not_expr | primary +primary ::= comparison | in_expr | motif_call | "(" expr ")" +comparison ::= field comp_op value +in_expr ::= field "IN" "[" value_list "]" +motif_call ::= "motif" "(" IDENTIFIER ")" +field ::= IDENTIFIER ("." IDENTIFIER)* +comp_op ::= "=" | "!=" | "<" | "<=" | ">" | ">=" +value ::= NUMBER | STRING +value_list ::= value ("," value)* +``` + +## Operator Precedence + +From lowest to highest: + +1. `OR` +2. `AND` +3. `NOT` (unary prefix) +4. Parenthesized expressions + +`AND` binds tighter than `OR`, so `a OR b AND c` is parsed as `a OR (b AND c)`. + +## Fields + +Dotted field names are mapped to database columns: + +| ChessQL Field | DB Column | Type | +|------------------|------------------|---------| +| `white.elo` | `white_elo` | INT | +| `black.elo` | `black_elo` | INT | +| `white.username` | `white_username` | VARCHAR | +| `black.username` | `black_username` | VARCHAR | +| `time.class` | `time_class` | VARCHAR | +| `num.moves` | `num_moves` | INT | +| `game.url` | `game_url` | VARCHAR | +| `played.at` | `played_at` | TIMESTAMP | +| `eco` | `eco` | VARCHAR | +| `result` | `result` | VARCHAR | +| `platform` | `platform` | VARCHAR | + +Underscore-separated names also work directly: `white_elo >= 2500` is equivalent to `white.elo >= 2500`. + +## Motifs + +The `motif()` function checks boolean columns for tactical pattern presence: + +| ChessQL | SQL | +|----------------------|------------------------------| +| `motif(pin)` | `has_pin = TRUE` | +| `motif(cross_pin)` | `has_cross_pin = TRUE` | +| `motif(fork)` | `has_fork = TRUE` | +| `motif(skewer)` | `has_skewer = TRUE` | +| `motif(discovered_attack)` | `has_discovered_attack = TRUE` | + +## Values + +- **Numbers**: integer literals, optionally negative. Examples: `2500`, `-1`, `0` +- **Strings**: double-quoted. Backslash escapes supported. Examples: `"chess.com"`, `"B90"`, `"hikaru"` + +## Examples + +### Simple comparisons + +``` +white.elo >= 2500 +eco = "B90" +num.moves > 40 +``` + +### Motif queries + +``` +motif(fork) +motif(cross_pin) +NOT motif(pin) +``` + +### Boolean combinations + +``` +white.elo >= 2500 AND motif(cross_pin) +motif(fork) OR motif(skewer) +motif(fork) AND NOT motif(pin) +``` + +### IN expressions + +``` +platform IN ["lichess", "chess.com"] +eco IN ["B90", "B91", "B92"] +``` + +### Complex queries + +``` +white.elo >= 2500 AND motif(fork) AND NOT motif(pin) +(motif(fork) OR motif(skewer)) AND white.elo > 2000 +platform IN ["chess.com"] AND black.elo > 2700 AND motif(discovered_attack) +``` + +## Compilation Examples + +| ChessQL Input | SQL Output | Parameters | +|---------------|-----------|------------| +| `white.elo >= 2500` | `white_elo >= ?` | `[2500]` | +| `motif(fork)` | `has_fork = TRUE` | `[]` | +| `white.elo >= 2500 AND motif(fork)` | `(white_elo >= ? AND has_fork = TRUE)` | `[2500]` | +| `motif(fork) OR motif(pin)` | `(has_fork = TRUE OR has_pin = TRUE)` | `[]` | +| `NOT motif(pin)` | `(NOT has_pin = TRUE)` | `[]` | +| `platform IN ["lichess", "chess.com"]` | `platform IN (?, ?)` | `["lichess", "chess.com"]` | +| `(motif(fork) OR motif(pin)) AND white.elo > 2000` | `((has_fork = TRUE OR has_pin = TRUE) AND white_elo > ?)` | `[2000]` | + +## Error Handling + +- **Unknown field**: `IllegalArgumentException` — "Unknown field: X" +- **Unknown motif**: `IllegalArgumentException` — "Unknown motif: X" +- **Syntax error**: `ParseException` — includes position information +- **Unterminated string**: `IllegalArgumentException` — includes position +- **Unexpected token**: `ParseException` — includes token and position + +## Security + +All values are bound as JDBC parameters (`?`), never interpolated into SQL strings. Field names and motif names are validated against a whitelist before being included in SQL. The compiler rejects any unrecognized identifiers. diff --git a/jvm/src/main/java/com/muchq/indexer/docs/DESIGN.md b/jvm/src/main/java/com/muchq/indexer/docs/DESIGN.md new file mode 100644 index 00000000..91062e29 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/docs/DESIGN.md @@ -0,0 +1,202 @@ +# Chess Game Indexer — System Design + +## Overview + +A Micronaut service that indexes chess games from chess.com (lichess planned), extracts tactical motifs via position replay, and exposes a custom query language (ChessQL) for searching indexed games. + +**Stack**: Java 21, Micronaut 4.x, Bazel, PostgreSQL, chariot (chess library), HikariCP + +## Architecture + +``` + ┌──────────────┐ + │ Client │ + └──────┬───────┘ + │ + ┌───────────▼────────────┐ + │ Micronaut HTTP │ + │ (Netty + JAX-RS) │ + ├────────────┬───────────┤ + │ IndexCtrl │ QueryCtrl │ + └─────┬──────┴─────┬─────┘ + │ │ + ┌────────▼──┐ ┌─────▼──────────┐ + │IndexQueue │ │ ChessQL │ + │(in-memory)│ │ Lexer→Parser→ │ + └────┬──────┘ │ Compiler→SQL │ + │ └─────┬───────────┘ + ┌─────▼──────┐ │ + │IndexWorker │ │ + │ (daemon) │ │ + └─────┬──────┘ │ + │ │ + ┌──────────▼─────┐ │ + │ FeatureExtract │ │ + │ PgnParser │ │ + │ GameReplayer │ │ + │ MotifDetect[] │ │ + └──────┬─────────┘ │ + │ │ + ┌─────▼────────────────────▼──┐ + │ PostgreSQL │ + │ indexing_requests │ + │ game_features │ + └─────────────────────────────┘ +``` + +## Package Structure + +``` +com.muchq.indexer/ + App.java Micronaut entry point + IndexerModule.java @Factory — wires all beans + + api/ + IndexController.java POST /index, GET /index/{id} + QueryController.java POST /query + + api/dto/ + IndexRequest.java Inbound: player, platform, month range + IndexResponse.java Outbound: id, status, gamesIndexed, error + QueryRequest.java Inbound: ChessQL query string, limit, offset + QueryResponse.java Outbound: list of GameFeatureRow, count + GameFeatureRow.java Projection of game_features for API consumers + + queue/ + IndexQueue.java Interface: enqueue, poll, size + IndexMessage.java Queue payload record + InMemoryIndexQueue.java LinkedBlockingQueue implementation + + worker/ + IndexWorker.java Processes IndexMessages: fetch→parse→detect→store + IndexWorkerLifecycle.java Daemon thread started on ServerStartupEvent + + db/ + DataSourceFactory.java HikariCP DataSource builder + Migration.java DDL bootstrap (CREATE TABLE IF NOT EXISTS) + IndexingRequestDao.java CRUD for indexing_requests + GameFeatureDao.java Insert + parameterized query for game_features + + engine/ + PgnParser.java Extracts headers + movetext from PGN strings + GameReplayer.java Uses chariot to replay moves → List + FeatureExtractor.java Orchestrates replay + all motif detectors + + engine/model/ + ParsedGame.java Headers map + movetext string + GameFeatures.java Set, numMoves, occurrence details + Motif.java Enum: PIN, CROSS_PIN, FORK, SKEWER, DISCOVERED_ATTACK + PositionContext.java moveNumber, FEN, whiteToMove + + motifs/ + MotifDetector.java Interface: motif(), detect(positions) + PinDetector.java Ray-casting from king to find pinned pieces + CrossPinDetector.java Piece pinned along two axes simultaneously + ForkDetector.java Piece attacking 2+ valuable enemy pieces + SkewerDetector.java Sliding attack through a more valuable piece + DiscoveredAttackDetector.java Piece moves to reveal sliding attacker behind it + + chessql/ + lexer/ TokenType, Token, Lexer + ast/ Expr (sealed), OrExpr, AndExpr, NotExpr, ComparisonExpr, InExpr, MotifExpr + parser/ Parser (recursive descent), ParseException + compiler/ SqlCompiler, CompiledQuery +``` + +## Data Flow + +### Indexing + +1. Client sends `POST /index` with player, platform, month range +2. `IndexController` creates a row in `indexing_requests` (status=PENDING), enqueues an `IndexMessage` +3. `IndexWorkerLifecycle` daemon thread polls the queue +4. `IndexWorker.process()`: + - Sets status to PROCESSING + - Iterates months, fetches games from chess.com API via `ChessClient` + - For each game: `FeatureExtractor.extract(pgn)` → `PgnParser` → `GameReplayer` → `MotifDetector[]` + - Inserts `GameFeatureRow` into `game_features` (ON CONFLICT DO NOTHING for idempotency) + - Updates `games_indexed` count periodically + - Sets status to COMPLETED or FAILED + +### Querying + +1. Client sends `POST /query` with a ChessQL string, limit, offset +2. `QueryController` parses ChessQL → AST → compiles to parameterized SQL +3. `GameFeatureDao.query()` executes the SQL against `game_features` +4. Results mapped to API DTOs and returned + +## DB Schema + +### indexing_requests + +| Column | Type | Notes | +|---------------|--------------|--------------------------------| +| id | UUID PK | gen_random_uuid() | +| player | VARCHAR(255) | chess.com username | +| platform | VARCHAR(50) | "chess.com", "lichess" (future)| +| start_month | VARCHAR(7) | "2024-01" | +| end_month | VARCHAR(7) | "2024-03" | +| status | VARCHAR(20) | PENDING→PROCESSING→COMPLETED/FAILED | +| created_at | TIMESTAMP | Immutable | +| updated_at | TIMESTAMP | Updated on status change | +| error_message | TEXT | Populated on FAILED | +| games_indexed | INT | Running count during processing| + +### game_features + +| Column | Type | Notes | +|-----------------------|---------------|----------------------------------------| +| id | UUID PK | gen_random_uuid() | +| request_id | UUID FK | References indexing_requests(id) | +| game_url | VARCHAR(1024) | UNIQUE — deduplication key | +| platform | VARCHAR(50) | | +| white_username | VARCHAR(255) | | +| black_username | VARCHAR(255) | | +| white_elo | INT | | +| black_elo | INT | | +| time_class | VARCHAR(50) | bullet, blitz, rapid, classical | +| eco | VARCHAR(10) | ECO opening code | +| result | VARCHAR(20) | win, checkmated, stalemate, etc. | +| played_at | TIMESTAMP | | +| num_moves | INT | | +| has_pin | BOOLEAN | Indexed for fast query | +| has_cross_pin | BOOLEAN | | +| has_fork | BOOLEAN | | +| has_skewer | BOOLEAN | | +| has_discovered_attack | BOOLEAN | | +| motifs_json | JSONB | Detailed occurrence data | +| pgn | TEXT | Full PGN for re-analysis | + +Boolean columns enable fast indexed queries. JSONB stores detailed motif occurrence data (move numbers, descriptions) for drill-down. + +## Configuration + +Environment variables (with defaults): + +| Variable | Default | +|------------------------|-------------------------------------------| +| `PORT` | 8080 | +| `APP_NAME` | helloworld (shared application.yml) | +| `INDEXER_DB_URL` | jdbc:postgresql://localhost:5432/indexer | +| `INDEXER_DB_USERNAME` | indexer | +| `INDEXER_DB_PASSWORD` | indexer | + +## Build & Test + +```bash +# Build everything +bazel build //jvm/src/main/java/com/muchq/indexer/... + +# Run all tests +bazel test //jvm/src/test/java/com/muchq/indexer/... + +# Run specific test suites +bazel test //jvm/src/test/java/com/muchq/indexer/chessql/lexer:LexerTest +bazel test //jvm/src/test/java/com/muchq/indexer/chessql/parser:ParserTest +bazel test //jvm/src/test/java/com/muchq/indexer/chessql/compiler:SqlCompilerTest +bazel test //jvm/src/test/java/com/muchq/indexer/engine:PgnParserTest +bazel test //jvm/src/test/java/com/muchq/indexer/queue:InMemoryIndexQueueTest + +# Build OCI image +bazel build //jvm/src/main/java/com/muchq/indexer:indexer_image +``` diff --git a/jvm/src/main/java/com/muchq/indexer/docs/DOCDB_STORAGE.md b/jvm/src/main/java/com/muchq/indexer/docs/DOCDB_STORAGE.md new file mode 100644 index 00000000..20ebe0ef --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/docs/DOCDB_STORAGE.md @@ -0,0 +1,983 @@ +# Chess Game Indexer — MongoDB / DocumentDB Storage Backend + +## Overview + +Replace PostgreSQL with MongoDB (or AWS DocumentDB, which implements the MongoDB 5.0 wire protocol) as the primary storage backend. The document model is a natural fit for game data: each game is a self-contained document with nested motif occurrence data, eliminating the JSONB column and the relational impedance mismatch. + +## Motivation + +| Concern | PostgreSQL | MongoDB / DocumentDB | +|---------|-----------|---------------------| +| Schema evolution | ALTER TABLE migrations | Schemaless — add fields freely | +| Motif detail storage | JSONB column (second-class) | Native nested documents | +| Horizontal scaling | Read replicas, partitioning | Built-in sharding | +| Cold storage tiering | Manual partition management | TTL indexes, automatic expiry | +| Retention policies | Custom worker + cron | Native TTL indexes | +| Query flexibility | SQL (powerful but rigid schema) | MQL (flexible, document-native) | +| Ops complexity (AWS) | RDS — managed but schema-bound | DocumentDB — managed, elastic | +| Cost at scale | RDS instances are fixed-size | DocumentDB scales storage independently | + +DocumentDB is especially attractive if the deployment is already AWS-native, since it provides MongoDB API compatibility with Aurora-style storage (replicated, durable, auto-scaling storage volume). + +## Document Model + +### indexing_requests Collection + +```json +{ + "_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "player": "hikaru", + "platform": "chess.com", + "startMonth": "2024-03", + "endMonth": "2024-03", + "status": "COMPLETED", + "createdAt": {"$date": "2024-03-15T10:00:00Z"}, + "updatedAt": {"$date": "2024-03-15T10:05:00Z"}, + "errorMessage": null, + "gamesIndexed": 147 +} +``` + +### game_features Collection + +```json +{ + "_id": "f1e2d3c4-b5a6-7890-abcd-ef1234567890", + "requestId": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "gameUrl": "https://www.chess.com/game/live/12345", + "platform": "chess.com", + "white": { + "username": "hikaru", + "elo": 2850 + }, + "black": { + "username": "magnuscarlsen", + "elo": 2830 + }, + "timeClass": "blitz", + "eco": "B90", + "result": "win", + "playedAt": {"$date": "2024-03-15T18:30:00Z"}, + "numMoves": 42, + "motifs": { + "pin": true, + "crossPin": false, + "fork": true, + "skewer": false, + "discoveredAttack": false + }, + "motifOccurrences": { + "pin": [ + {"moveNumber": 15, "description": "Pin detected at move 15"} + ], + "fork": [ + {"moveNumber": 23, "description": "Fork detected at move 23"}, + {"moveNumber": 31, "description": "Fork detected at move 31"} + ] + }, + "pgn": "[Event \"Live Chess\"]..." +} +``` + +Key differences from the relational model: +- `white` and `black` are nested objects (not flat `white_username`, `white_elo` columns) +- `motifs` is a nested object with boolean fields (not top-level `has_*` columns) +- `motifOccurrences` replaces the `motifs_json` JSONB column with a native nested structure +- No foreign key constraint — `requestId` is a string reference, enforced at the application level + +### Indexes + +```javascript +// game_features collection +db.game_features.createIndex({ "gameUrl": 1 }, { unique: true }); +db.game_features.createIndex({ "requestId": 1 }); +db.game_features.createIndex({ "white.username": 1 }); +db.game_features.createIndex({ "black.username": 1 }); +db.game_features.createIndex({ "white.elo": 1 }); +db.game_features.createIndex({ "black.elo": 1 }); +db.game_features.createIndex({ "motifs.pin": 1 }); +db.game_features.createIndex({ "motifs.fork": 1 }); +db.game_features.createIndex({ "motifs.skewer": 1 }); +db.game_features.createIndex({ "motifs.crossPin": 1 }); +db.game_features.createIndex({ "motifs.discoveredAttack": 1 }); +db.game_features.createIndex({ "eco": 1 }); +db.game_features.createIndex({ "timeClass": 1 }); +db.game_features.createIndex({ "playedAt": 1 }); +db.game_features.createIndex({ "platform": 1 }); + +// Compound indexes for common query patterns +db.game_features.createIndex({ "white.elo": 1, "motifs.fork": 1 }); +db.game_features.createIndex({ "black.elo": 1, "motifs.fork": 1 }); +db.game_features.createIndex({ "eco": 1, "motifs.pin": 1 }); + +// TTL index for automatic retention (see Retention section) +db.game_features.createIndex({ "playedAt": 1 }, { expireAfterSeconds: 7776000 }); // 90 days + +// indexing_requests collection +db.indexing_requests.createIndex({ "player": 1, "platform": 1, "startMonth": 1, "endMonth": 1 }); +db.indexing_requests.createIndex({ "status": 1 }); +``` + +--- + +## ChessQL Compiler Changes + +The `SqlCompiler` currently emits parameterized SQL. A new `MongoCompiler` emits MongoDB query documents (`org.bson.Document` / `Bson` filters). + +### New Class: MongoCompiler + +```java +public class MongoCompiler { + public Bson compile(Expr expr) { + return compileExpr(expr); + } + + private Bson compileExpr(Expr expr) { + return switch (expr) { + case OrExpr or -> Filters.or( + or.operands().stream().map(this::compileExpr).toList() + ); + case AndExpr and -> Filters.and( + and.operands().stream().map(this::compileExpr).toList() + ); + case NotExpr not -> Filters.not(compileExpr(not.operand())); + case ComparisonExpr cmp -> compileComparison(cmp); + case InExpr in -> Filters.in(resolveField(in.field()), in.values()); + case MotifExpr motif -> Filters.eq(resolveMotifField(motif.motifName()), true); + }; + } + + private Bson compileComparison(ComparisonExpr cmp) { + String field = resolveField(cmp.field()); + Object value = cmp.value(); + return switch (cmp.operator()) { + case "=" -> Filters.eq(field, value); + case "!=" -> Filters.ne(field, value); + case "<" -> Filters.lt(field, value); + case "<=" -> Filters.lte(field, value); + case ">" -> Filters.gt(field, value); + case ">=" -> Filters.gte(field, value); + default -> throw new IllegalArgumentException("Invalid operator: " + cmp.operator()); + }; + } +} +``` + +### Field Mapping Changes + +The document model uses nested paths instead of flat column names: + +| ChessQL Field | PostgreSQL Column | MongoDB Field Path | +|------------------|--------------------|----------------------------| +| `white.elo` | `white_elo` | `white.elo` | +| `black.elo` | `black_elo` | `black.elo` | +| `white.username` | `white_username` | `white.username` | +| `black.username` | `black_username` | `black.username` | +| `time.class` | `time_class` | `timeClass` | +| `num.moves` | `num_moves` | `numMoves` | +| `game.url` | `game_url` | `gameUrl` | +| `played.at` | `played_at` | `playedAt` | +| `eco` | `eco` | `eco` | +| `result` | `result` | `result` | +| `platform` | `platform` | `platform` | + +Motif field mapping: + +| ChessQL | PostgreSQL | MongoDB | +|----------------------|-----------------------------|------------------------------| +| `motif(pin)` | `has_pin = TRUE` | `motifs.pin = true` | +| `motif(cross_pin)` | `has_cross_pin = TRUE` | `motifs.crossPin = true` | +| `motif(fork)` | `has_fork = TRUE` | `motifs.fork = true` | +| `motif(skewer)` | `has_skewer = TRUE` | `motifs.skewer = true` | +| `motif(discovered_attack)` | `has_discovered_attack = TRUE` | `motifs.discoveredAttack = true` | + +The dotted ChessQL field syntax (`white.elo`) maps directly to MongoDB's dot-notation for nested documents. This is a natural fit — the ChessQL grammar was accidentally designed for document databases. + +### Compiler Interface + +Abstract the compilation target behind an interface so both backends coexist: + +```java +public interface QueryCompiler { + T compile(Expr expr); +} + +public class SqlCompiler implements QueryCompiler { ... } +public class MongoCompiler implements QueryCompiler { ... } +``` + +The `QueryController` selects the compiler based on the active storage backend. + +--- + +## DAO Changes + +### New Interface: GameFeatureStore + +Abstract the storage layer so PostgreSQL and MongoDB implementations are swappable: + +```java +public interface GameFeatureStore { + void insert(GameFeatureDocument doc); + List query(Object compiledQuery, int limit, int offset); +} + +public interface IndexingRequestStore { + String create(String player, String platform, String startMonth, String endMonth); + Optional findById(String id); + void updateStatus(String id, String status, String errorMessage, int gamesIndexed); +} +``` + +### MongoGameFeatureStore + +```java +public class MongoGameFeatureStore implements GameFeatureStore { + private final MongoCollection collection; + + public MongoGameFeatureStore(MongoDatabase database) { + this.collection = database.getCollection("game_features"); + } + + @Override + public void insert(GameFeatureDocument doc) { + Document mongoDoc = toDocument(doc); + collection.insertOne(mongoDoc); + // For idempotency: catch DuplicateKeyException (unique index on gameUrl) + } + + @Override + public List query(Object compiledQuery, int limit, int offset) { + Bson filter = (Bson) compiledQuery; + return collection.find(filter) + .skip(offset) + .limit(limit) + .map(this::fromDocument) + .into(new ArrayList<>()); + } +} +``` + +### MongoIndexingRequestStore + +```java +public class MongoIndexingRequestStore implements IndexingRequestStore { + private final MongoCollection collection; + + @Override + public String create(String player, String platform, String startMonth, String endMonth) { + String id = UUID.randomUUID().toString(); + Document doc = new Document() + .append("_id", id) + .append("player", player) + .append("platform", platform) + .append("startMonth", startMonth) + .append("endMonth", endMonth) + .append("status", "PENDING") + .append("createdAt", new Date()) + .append("updatedAt", new Date()) + .append("gamesIndexed", 0); + collection.insertOne(doc); + return id; + } + + @Override + public void updateStatus(String id, String status, String errorMessage, int gamesIndexed) { + collection.updateOne( + Filters.eq("_id", id), + Updates.combine( + Updates.set("status", status), + Updates.set("errorMessage", errorMessage), + Updates.set("gamesIndexed", gamesIndexed), + Updates.set("updatedAt", new Date()) + ) + ); + } +} +``` + +--- + +## DocumentDB-Specific Considerations + +AWS DocumentDB implements MongoDB 5.0 API with some differences. These require attention: + +### Supported Features (no changes needed) + +- CRUD operations (insertOne, find, updateOne, deleteMany) +- Query filters (Filters.eq, gt, gte, lt, lte, ne, in, and, or, not) +- Indexes (single field, compound, unique, TTL) +- Skip / limit pagination +- Dot-notation for nested documents +- `$set` update operator + +### Unsupported or Limited Features + +| Feature | MongoDB | DocumentDB | Workaround | +|---------|---------|------------|------------| +| `$jsonSchema` validation | Full support | Not supported | Validate in application layer | +| Change streams | Full support | Supported (with limitations on sharded collections) | Use polling if change streams are unreliable | +| Transactions (multi-doc) | Full support | Supported on 4.0+ compatible clusters | Use single-document operations where possible | +| `$merge` aggregation | Supported | Not supported | Use application-side read + write for cold storage export | +| `$out` aggregation | Supported | Limited | Same as above | +| `$lookup` (joins) | Supported | Supported but slower | Denormalize — the document model already avoids joins | +| Text search (`$text`) | Full text index | Not supported | Use regex or application-side filtering for text search | +| Collation | Full support | Partial | Use case-insensitive regex for username matching | +| Aggregation pipeline | Full support | Subset supported | Test specific pipeline stages against DocumentDB | + +### Required DocumentDB Cluster Configuration + +``` +Engine version: 5.0.0 (latest DocumentDB version) +Instance class: db.r6g.large (minimum for production) +Storage encryption: Enabled (AES-256) +TLS: Required (DocumentDB enforces TLS by default) +Cluster parameter group: + - tls: enabled + - audit_logs: enabled + - profiler: enabled (threshold: 100ms) +``` + +### Connection String + +``` +mongodb://:@:27017/?tls=true&tlsCAFile=global-bundle.pem&replicaSet=rs0&readPreference=secondaryPreferred&retryWrites=false +``` + +Key differences from standard MongoDB connections: +- **`retryWrites=false`**: DocumentDB does not support retryable writes. The application must handle write retries. +- **`tlsCAFile`**: DocumentDB requires TLS with the AWS-provided CA bundle. +- **`replicaSet=rs0`**: DocumentDB always uses replica set `rs0`. +- **`readPreference=secondaryPreferred`**: Route reads to replicas to reduce primary load. + +### Application-Side Retry for Writes + +Since DocumentDB does not support `retryWrites`, add retry logic to the store layer: + +```java +public class RetryableMongoStore { + private static final int MAX_RETRIES = 3; + + protected T withRetry(Supplier operation) { + for (int attempt = 1; attempt <= MAX_RETRIES; attempt++) { + try { + return operation.get(); + } catch (MongoException e) { + if (isRetryable(e) && attempt < MAX_RETRIES) { + sleep(100L * attempt); // linear backoff + continue; + } + throw e; + } + } + throw new IllegalStateException("Unreachable"); + } + + private boolean isRetryable(MongoException e) { + // Network errors, not-primary errors + return e.hasErrorLabel("RetryableWriteError") + || e.getCode() == 10107 // NotWritablePrimary + || e.getCode() == 13435; // NotPrimaryNoSecondaryOk + } +} +``` + +--- + +## Retention with TTL Indexes + +MongoDB/DocumentDB TTL indexes provide automatic document expiry — no retention worker needed for simple cases. + +### Hot → Delete (Simple) + +```javascript +// Documents deleted 90 days after playedAt +db.game_features.createIndex({ "playedAt": 1 }, { expireAfterSeconds: 7776000 }); +``` + +DocumentDB runs a background task every 60 seconds to remove expired documents. No application code needed. + +### Hot → Cold (With Archive) + +For the tiered retention policy from ROADMAP.md, TTL indexes handle the delete tier, but hot→cold migration still needs application logic: + +``` + TTL = none TTL = 90 days S3 lifecycle = 365 days + ┌──────────────────┐ Export ┌──────────────────┐ ┌──────────────┐ + │ game_features │ ──────────► │ S3 NDJSON.gz │ ────► │ Deleted │ + │ (DocumentDB) │ App logic │ (cold storage) │ Auto │ │ + │ Auto-expire │ └──────────────────┘ └──────────────┘ + │ after 90 days │ + └──────────────────┘ +``` + +The export step runs before TTL expiry: + +```java +public class ColdStorageExporter { + public void exportExpiringSoon(int daysBeforeExpiry) { + // Find documents expiring within N days + Instant cutoff = Instant.now().minus(Duration.ofDays(90 - daysBeforeExpiry)); + Bson filter = Filters.lt("playedAt", Date.from(cutoff)); + + // Stream to S3 as NDJSON + try (MongoCursor cursor = collection.find(filter).iterator()) { + while (cursor.hasNext()) { + s3Writer.writeLine(cursor.next().toJson()); + } + } + } +} +``` + +Run this daily, exporting documents that will expire in the next 7 days. TTL handles the actual deletion. + +### Adjusting TTL After Creation + +DocumentDB supports modifying TTL index expiry: + +```javascript +db.runCommand({ + collMod: "game_features", + index: { + keyPattern: { "playedAt": 1 }, + expireAfterSeconds: 15552000 // Change to 180 days + } +}); +``` + +--- + +## Migration Path from PostgreSQL + +### Phase 1: Dual-Write + +Write to both PostgreSQL and DocumentDB simultaneously. Read from PostgreSQL. This validates the DocumentDB schema and write path without risk. + +```java +public class DualWriteGameFeatureStore implements GameFeatureStore { + private final GameFeatureDao postgresDao; + private final MongoGameFeatureStore mongoStore; + + @Override + public void insert(GameFeatureDocument doc) { + postgresDao.insert(toRelationalRow(doc)); + try { + mongoStore.insert(doc); + } catch (Exception e) { + LOG.warn("DocumentDB write failed, PostgreSQL is authoritative", e); + } + } +} +``` + +### Phase 2: Dual-Read Validation + +Read from both and compare results. Log discrepancies. This validates the MongoCompiler produces equivalent results to SqlCompiler. + +### Phase 3: Switch Reads + +Point reads to DocumentDB. PostgreSQL is still written to as a fallback. + +### Phase 4: Remove PostgreSQL + +Stop writing to PostgreSQL. Remove the relational DAOs and SqlCompiler (or keep as an option via configuration). + +--- + +## Dependencies + +### Maven Artifacts + +``` +# bazel/java.MODULE.bazel +"org.mongodb:mongodb-driver-sync:5.1.0", +"org.mongodb:bson:5.1.0", +``` + +Bazel labels: +``` +@maven//:org_mongodb_mongodb_driver_sync +@maven//:org_mongodb_bson +``` + +### New BUILD Targets + +``` +# jvm/src/main/java/com/muchq/indexer/db/mongo/BUILD.bazel +load("@rules_java//java:java_library.bzl", "java_library") + +java_library( + name = "mongo", + srcs = [ + "MongoClientFactory.java", + "MongoGameFeatureStore.java", + "MongoIndexingRequestStore.java", + "MongoMigration.java", + ], + visibility = ["//visibility:public"], + deps = [ + "//jvm/src/main/java/com/muchq/indexer/db", + "@maven//:org_mongodb_mongodb_driver_sync", + "@maven//:org_mongodb_bson", + "@maven//:org_slf4j_slf4j_api", + ], +) +``` + +``` +# jvm/src/main/java/com/muchq/indexer/chessql/compiler/mongo/BUILD.bazel +load("@rules_java//java:java_library.bzl", "java_library") + +java_library( + name = "mongo", + srcs = ["MongoCompiler.java"], + visibility = ["//visibility:public"], + deps = [ + "//jvm/src/main/java/com/muchq/indexer/chessql/ast", + "@maven//:org_mongodb_mongodb_driver_sync", + "@maven//:org_mongodb_bson", + ], +) +``` + +--- + +## Configuration + +``` +INDEXER_STORAGE=postgres|docdb # Storage backend (default: postgres) +INDEXER_DOCDB_URI=mongodb://... # DocumentDB connection string +INDEXER_DOCDB_DATABASE=indexer # Database name +INDEXER_DOCDB_TLS_CA_FILE=/path/to/global-bundle.pem # AWS CA bundle +INDEXER_DOCDB_RETRY_WRITES=false # Must be false for DocumentDB +``` + +### IndexerModule Wiring + +```java +@Context +public GameFeatureStore gameFeatureStore( + @Value("${indexer.storage:postgres}") String storage, + ...) { + return switch (storage) { + case "docdb" -> { + MongoClient client = MongoClientFactory.create(docdbUri, caFile); + MongoDatabase db = client.getDatabase(database); + new MongoMigration(db).run(); // create indexes + yield new MongoGameFeatureStore(db); + } + default -> new PostgresGameFeatureStore(dataSource); + }; +} + +@Context +public QueryCompiler queryCompiler( + @Value("${indexer.storage:postgres}") String storage) { + return switch (storage) { + case "docdb" -> new MongoCompiler(); + default -> new SqlCompiler(); + }; +} +``` + +--- + +## Cost Comparison + +### Small (< 100K games) + +| Resource | PostgreSQL (RDS) | DocumentDB | +|----------|-----------------|------------| +| Instance | db.t4g.micro — $15/mo | db.t3.medium — $58/mo | +| Storage | 20GB gp3 — $2/mo | Auto-scaling — $0.10/GB ($2/mo) | +| I/O | Included in gp3 | $0.20 per 1M reads, $0.20 per 1M writes | +| Backup | Free (7 days) | Free (1 day), $0.02/GB beyond | +| **Total** | **~$17/mo** | **~$62/mo** | + +DocumentDB is ~3.5x more expensive at small scale due to higher minimum instance cost. **PostgreSQL wins here.** + +### Medium (100K - 1M games) + +| Resource | PostgreSQL (RDS) | DocumentDB | +|----------|-----------------|------------| +| Instance | db.t4g.medium — $50/mo | db.r6g.large — $195/mo | +| Storage | 100GB gp3 — $10/mo | 10GB auto — $1/mo | +| I/O | Included | ~$5/mo | +| Read replica | db.t4g.medium — $50/mo | db.r6g.large — $195/mo | +| **Total** | **~$110/mo** | **~$396/mo** | + +DocumentDB remains more expensive. However, schema evolution is free (no ALTER TABLE downtime), and TTL indexes eliminate the retention worker. **PostgreSQL still wins on cost, DocumentDB wins on operational simplicity.** + +### Large (1M+ games, multi-motif, frequent schema changes) + +| Resource | PostgreSQL (RDS) | DocumentDB | +|----------|-----------------|------------| +| Primary | db.r7g.large — $250/mo | db.r6g.xlarge — $390/mo | +| Replicas (2x) | $500/mo | $780/mo | +| Storage | 500GB gp3 — $50/mo | 50GB auto — $5/mo | +| I/O | Included | ~$30/mo | +| Partition management | Manual (ops time) | Automatic (TTL) | +| Schema migrations | Downtime risk | Zero-downtime | +| **Total** | **~$800/mo** | **~$1,205/mo** | + +DocumentDB is ~50% more expensive on compute but eliminates partition management, schema migration risk, and the retention worker. **Choose based on whether ops time or compute cost dominates your budget.** + +### When to Choose DocumentDB + +- Frequent schema changes (new motifs, new game metadata fields) +- AWS-native deployment where DocumentDB is already in use +- Team more familiar with MongoDB than PostgreSQL +- Retention policy is a hard requirement and TTL indexes remove significant ops burden +- Query patterns are primarily document lookups, not complex joins or aggregations + +### When to Stay on PostgreSQL + +- Cost-sensitive deployment +- Complex analytical queries beyond simple ChessQL (window functions, CTEs) +- Need JSONB querying within motif occurrence data +- Team more familiar with SQL +- Smaller dataset where schema migration is infrequent + +--- + +## Files to Create / Modify + +| File | Action | Description | +|------|--------|-------------| +| `bazel/java.MODULE.bazel` | Modify | Add MongoDB driver deps | +| `db/GameFeatureStore.java` | Create | Storage interface | +| `db/IndexingRequestStore.java` | Create | Storage interface | +| `db/PostgresGameFeatureStore.java` | Rename/refactor | Existing `GameFeatureDao` implements interface | +| `db/PostgresIndexingRequestStore.java` | Rename/refactor | Existing `IndexingRequestDao` implements interface | +| `db/mongo/MongoClientFactory.java` | Create | DocumentDB connection builder (TLS, CA bundle) | +| `db/mongo/MongoGameFeatureStore.java` | Create | MongoDB implementation of GameFeatureStore | +| `db/mongo/MongoIndexingRequestStore.java` | Create | MongoDB implementation of IndexingRequestStore | +| `db/mongo/MongoMigration.java` | Create | Index creation on startup | +| `chessql/compiler/QueryCompiler.java` | Create | Compiler interface | +| `chessql/compiler/mongo/MongoCompiler.java` | Create | ChessQL → Bson filter compiler | +| `IndexerModule.java` | Modify | Storage backend switch | +| `worker/IndexWorker.java` | Modify | Use `GameFeatureStore` interface | +| `api/QueryController.java` | Modify | Use `QueryCompiler` interface | +| `api/IndexController.java` | Modify | Use `IndexingRequestStore` interface | + +~8 new files, ~6 modified files, ~800 lines of new code. + +--- + +## Leveraging the Rust doc_db Service + +The repository contains an existing Rust-based document database service at `rust/doc_db/` that wraps MongoDB with a gRPC interface. This section analyzes its suitability for the chess indexer and documents required changes. + +### Current doc_db Architecture + +``` +┌─────────────────┐ gRPC ┌─────────────────┐ +│ Java Indexer │ ────────────► │ Rust doc_db │ +│ (client) │ │ (tonic server) │ +└─────────────────┘ └────────┬────────┘ + │ + ┌──────▼──────┐ + │ MongoDB │ + │ /DocumentDB│ + └─────────────┘ +``` + +### Current doc_db API (from `protos/doc_db/doc_db.proto`) + +```protobuf +service DocDb { + rpc InsertDoc (InsertDocRequest) returns (InsertDocResponse) {} + rpc UpdateDoc (UpdateDocRequest) returns (UpdateDocResponse) {} + rpc FindDocById (FindDocByIdRequest) returns (FindDocByIdResponse) {} + rpc FindDoc (FindDocRequest) returns (FindDocResponse) {} +} + +message Document { + string id = 1; + string version = 2; + bytes bytes = 3; + map tags = 4; +} +``` + +### Current doc_db Data Model + +| Field | Type | Purpose | +|-----------|------------------------|----------------------------------------------| +| `_id` | ObjectId | MongoDB document ID | +| `bytes` | `Vec` | Opaque payload (serialized document) | +| `version` | String (UUID) | Optimistic concurrency control | +| `tags` | `HashMap` | Queryable string key-value pairs | + +### Suitability Analysis + +| Requirement | Current doc_db Support | Gap | +|-------------|----------------------|-----| +| Store game documents | Partial — `bytes` field can hold serialized JSON | No native nested document support | +| Query by numeric fields (`white.elo >= 2500`) | **No** — tags are string equality only | Cannot express range queries | +| Query by motif booleans (`motif(fork)`) | Partial — could encode as tag `"has_fork": "true"` | String equality, not boolean | +| Complex boolean queries (AND/OR/NOT) | **No** — `FindDoc` matches exact tag set | No boolean combinators | +| Pagination (limit/offset) | **No** — returns single document | Cannot paginate results | +| Batch queries | **No** — `find_one` only | No `find_many` | +| TTL / retention | **No** — no TTL support | Would need MongoDB-side TTL index + bypass doc_db | +| Indexing | **No** — no index management | Must create indexes out-of-band | + +**Verdict**: The current doc_db API is designed for simple key-value / tag-based document storage with optimistic locking. It is **not suitable** for the chess indexer's ChessQL query requirements without significant extensions. + +### Required doc_db Extensions + +To support the chess indexer, doc_db would need the following additions: + +#### 1. Rich Query Support + +New RPC for MongoDB-style queries: + +```protobuf +message QueryRequest { + string collection = 1; + bytes filter_bson = 2; // Serialized BSON filter document + int32 limit = 3; + int32 skip = 4; + bytes sort_bson = 5; // Optional sort specification +} + +message QueryResponse { + repeated Document docs = 1; + int64 total_count = 2; // For pagination UI +} + +service DocDb { + // ... existing RPCs ... + rpc Query (QueryRequest) returns (QueryResponse) {} +} +``` + +The `filter_bson` field would accept a raw BSON-serialized MongoDB query document, allowing the Java client to build arbitrary queries using the MongoDB Java driver's `Filters` API and serialize to BSON. + +**Rust implementation sketch**: + +```rust +async fn query(&self, request: Request) -> Result, Status> { + let req = request.into_inner(); + let db_name = read_db_name_from_metadata(request.metadata())?; + + let filter: BsonDocument = bson::from_slice(&req.filter_bson) + .map_err(|e| Status::invalid_argument(format!("invalid filter: {}", e)))?; + + let collection = self.client.database(&db_name).collection::(&req.collection); + + let options = FindOptions::builder() + .limit(req.limit as i64) + .skip(req.skip as u64) + .build(); + + let mut cursor = collection.find(filter, options).await + .map_err(|e| Status::internal(e.to_string()))?; + + let mut docs = Vec::new(); + while let Some(doc) = cursor.try_next().await? { + docs.push(to_proto_document(doc)); + } + + Ok(Response::new(QueryResponse { docs, total_count: docs.len() as i64 })) +} +``` + +#### 2. Batch Insert + +For efficient game indexing: + +```protobuf +message BatchInsertRequest { + string collection = 1; + repeated DocumentEgg docs = 2; +} + +message BatchInsertResponse { + repeated string ids = 1; + int32 inserted_count = 2; +} + +service DocDb { + rpc BatchInsert (BatchInsertRequest) returns (BatchInsertResponse) {} +} +``` + +#### 3. Index Management + +```protobuf +message CreateIndexRequest { + string collection = 1; + bytes index_spec_bson = 2; // e.g., {"white.elo": 1} + IndexOptions options = 3; +} + +message IndexOptions { + bool unique = 1; + int64 expire_after_seconds = 2; // For TTL indexes + string name = 3; +} + +message CreateIndexResponse { + string index_name = 1; +} + +service DocDb { + rpc CreateIndex (CreateIndexRequest) returns (CreateIndexResponse) {} +} +``` + +#### 4. Native Document Storage (Alternative to bytes) + +Instead of serializing to `bytes`, store native BSON documents: + +```protobuf +message RichDocument { + string id = 1; + string version = 2; + bytes bson_content = 3; // Full BSON document, not just payload +} + +message InsertRichDocRequest { + string collection = 1; + bytes bson_content = 2; // Client serializes full document +} +``` + +This allows MongoDB to index nested fields directly without the doc_db service needing to understand the schema. + +### Integration Architecture Options + +#### Option A: Extend doc_db (Recommended if Rust investment is desired) + +``` +┌─────────────────┐ gRPC ┌─────────────────┐ +│ Java Indexer │ ────────────► │ Rust doc_db │ +│ │ │ (extended) │ +│ - ChessQL │ │ - Query() │ +│ - MongoCompiler│ │ - BatchInsert()│ +│ → BSON bytes │ │ - CreateIndex()│ +└─────────────────┘ └────────┬────────┘ + │ + ┌──────▼──────┐ + │ DocumentDB │ + └─────────────┘ +``` + +**Pros**: +- Single MongoDB connection pool managed by Rust service +- Rust service can add caching, rate limiting, connection management +- Language-agnostic — other services can use the gRPC API +- Existing doc_db codebase provides foundation + +**Cons**: +- Requires Rust development for new RPCs +- Additional network hop (Java → gRPC → MongoDB) +- BSON serialization/deserialization overhead at both ends +- More complex deployment (two services) + +#### Option B: Direct MongoDB from Java (Current doc recommendation) + +``` +┌─────────────────┐ +│ Java Indexer │ +│ │ +│ - ChessQL │──────────────────┐ +│ - MongoCompiler│ │ +│ - MongoDriver │ │ +└─────────────────┘ │ + ┌──────▼──────┐ + │ DocumentDB │ + └─────────────┘ +``` + +**Pros**: +- No additional service to deploy +- Native MongoDB driver has full feature support +- Lower latency (no gRPC hop) +- Simpler debugging + +**Cons**: +- Each Java service manages its own connection pool +- MongoDB-specific code in Java (less portable) + +#### Option C: Hybrid — Use doc_db for Simple Ops, Direct for Queries + +``` +┌─────────────────┐ gRPC ┌─────────────────┐ +│ Java Indexer │ ────────────► │ Rust doc_db │ ← InsertDoc, UpdateDoc +│ │ │ (unchanged) │ +│ - ChessQL │ └────────┬────────┘ +│ - MongoCompiler│──────────────────┐ │ +│ - MongoDriver │ │ │ +└─────────────────┘ │ │ + ┌──────▼─────▼┐ + │ DocumentDB │ + └─────────────┘ +``` + +Use doc_db for writes (with optimistic locking), direct MongoDB driver for complex queries. This avoids extending doc_db while leveraging its write-side features. + +### Estimated Changes to doc_db + +| Change | Complexity | Lines of Rust | +|--------|-----------|---------------| +| Add `Query` RPC with BSON filter | Medium | ~100 | +| Add `BatchInsert` RPC | Low | ~50 | +| Add `CreateIndex` RPC | Low | ~40 | +| Add pagination to `FindDoc` | Low | ~30 | +| Proto file updates | Low | ~50 | +| Tests for new RPCs | Medium | ~200 | +| **Total** | | **~470 lines** | + +### Java Client for Extended doc_db + +Generate Java gRPC stubs from the updated proto: + +``` +# bazel/java.MODULE.bazel +"io.grpc:grpc-netty-shaded:1.62.2", +"io.grpc:grpc-protobuf:1.62.2", +"io.grpc:grpc-stub:1.62.2", +``` + +```java +public class DocDbGameFeatureStore implements GameFeatureStore { + private final DocDbGrpc.DocDbBlockingStub stub; + private final ObjectMapper mapper; + + @Override + public List query(Object compiledQuery, int limit, int offset) { + Bson bsonFilter = (Bson) compiledQuery; + byte[] filterBytes = toBsonBytes(bsonFilter); + + QueryRequest request = QueryRequest.newBuilder() + .setCollection("game_features") + .setFilterBson(ByteString.copyFrom(filterBytes)) + .setLimit(limit) + .setSkip(offset) + .build(); + + QueryResponse response = stub.query(request); + return response.getDocsList().stream() + .map(this::fromProtoDocument) + .toList(); + } +} +``` + +### Recommendation + +**For the chess indexer specifically**: Use **Option B (Direct MongoDB from Java)** as documented in the main sections above. The doc_db service's current API is not a good fit, and extending it adds complexity without significant benefit for a single-consumer scenario. + +**Consider extending doc_db if**: +- Multiple services (not just the indexer) need MongoDB access +- You want to centralize connection management, caching, and rate limiting +- The team prefers Rust for database interaction logic +- You need a language-agnostic document API for future polyglot services + +**Keep doc_db as-is for**: +- Its original use case (simple document storage with optimistic locking) +- Services that only need tag-based lookup (not range queries) +- Write-heavy workloads where the version-based update pattern is valuable diff --git a/jvm/src/main/java/com/muchq/indexer/docs/IN_PROCESS_MODE.md b/jvm/src/main/java/com/muchq/indexer/docs/IN_PROCESS_MODE.md new file mode 100644 index 00000000..ee4f0f8d --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/docs/IN_PROCESS_MODE.md @@ -0,0 +1,345 @@ +# Chess Game Indexer — In-Process Mode + +## Quick Start + +```bash +# Start the service (H2 in-memory, no external dependencies) +INDEXER_DB_URL="jdbc:h2:mem:indexer;DB_CLOSE_DELAY=-1" bazel run //jvm/src/main/java/com/muchq/indexer:indexer + +# Index a player's games +curl -X POST http://localhost:8080/index \ + -H 'Content-Type: application/json' \ + -d '{"player":"hikaru","platform":"CHESS_COM","startMonth":"2026-01","endMonth":"2026-01"}' + +# Check indexing status (replace {id} with the returned ID) +curl http://localhost:8080/index/{id} + +# Query indexed games using ChessQL +curl -X POST http://localhost:8080/query \ + -H 'Content-Type: application/json' \ + -d '{"query":"white.elo > 2500","limit":10,"offset":0}' + +# Query with motif detection +curl -X POST http://localhost:8080/query \ + -H 'Content-Type: application/json' \ + -d '{"query":"motif(fork) AND motif(pin)","limit":10,"offset":0}' +``` + +## Overview + +**In-process mode is the default.** The indexer runs with no external dependencies — no PostgreSQL, no SQS, no S3. Everything lives in-process using an in-memory queue and an H2 in-memory database. + +This mode is useful for: +- Local development without Docker or PostgreSQL installed +- Unit and integration testing without test containers +- CLI tooling (index a player, query results, exit) +- Demos and evaluations +- CI pipelines + +To use PostgreSQL instead, set the `INDEXER_DB_URL` environment variable to a PostgreSQL JDBC URL (e.g., `jdbc:postgresql://localhost:5432/indexer`). + +## Architecture + +``` +┌──────────────────────────────────────────────┐ +│ JVM Process │ +│ │ +│ ┌───────────┐ ┌──────────────────────┐ │ +│ │ HTTP API │ │ InMemoryIndexQueue │ │ +│ │ /index ├──►│ (LinkedBlockingQueue) │ │ +│ │ /query │ └──────────┬───────────┘ │ +│ └─────┬─────┘ │ │ +│ │ ┌─────▼──────┐ │ +│ │ │IndexWorker │ │ +│ │ └─────┬──────┘ │ +│ │ │ │ +│ │ ┌───────────────▼──────────────┐ │ +│ └───►│ H2 (in-memory) │ │ +│ │ indexing_requests │ │ +│ │ game_features │ │ +│ └──────────────────────────────┘ │ +└──────────────────────────────────────────────┘ +``` + +Zero external processes. Start the JAR, use it, stop it. Data lives only for the lifetime of the process. + +## Implementation Plan + +### 1. Add H2 Dependency + +H2 is a pure-Java SQL database that runs embedded. It supports a large subset of PostgreSQL syntax. + +``` +# bazel/java.MODULE.bazel +"com.h2database:h2:2.2.224", +``` + +Bazel label: `@maven//:com_h2database_h2` + +### 2. Adapt DataSourceFactory + +```java +public class DataSourceFactory { + public static DataSource create(String jdbcUrl, String username, String password) { + // Existing HikariCP path — works for both H2 and PostgreSQL + HikariConfig config = new HikariConfig(); + config.setJdbcUrl(jdbcUrl); + config.setUsername(username); + config.setPassword(password); + config.setMaximumPoolSize(10); + config.setMinimumIdle(2); + return new HikariDataSource(config); + } + + public static DataSource createInMemory() { + return create("jdbc:h2:mem:indexer;MODE=PostgreSQL;DB_CLOSE_DELAY=-1", "sa", ""); + } +} +``` + +Key H2 JDBC URL flags: +- `mem:indexer` — named in-memory database +- `MODE=PostgreSQL` — PostgreSQL compatibility (boolean handling, function names) +- `DB_CLOSE_DELAY=-1` — keep DB alive as long as the JVM runs (default is to drop when last connection closes) + +### 3. Adapt Migration.java for H2 Compatibility + +The existing DDL uses `gen_random_uuid()` and `JSONB`, which H2 does not support in PostgreSQL mode. The migration needs dialect-aware DDL: + +```java +public class Migration { + private final DataSource dataSource; + private final boolean isH2; + + public Migration(DataSource dataSource) { + this.dataSource = dataSource; + this.isH2 = detectH2(dataSource); + } + + public void run() { + if (isH2) { + runH2Schema(); + } else { + runPostgresSchema(); + } + } + + private void runH2Schema() { + // UUID PRIMARY KEY DEFAULT random_uuid() + // TEXT instead of JSONB + // No gen_random_uuid() — use random_uuid() + } + + private boolean detectH2(DataSource ds) { + try (Connection conn = ds.getConnection()) { + return conn.getMetaData().getDatabaseProductName().contains("H2"); + } + } +} +``` + +**Schema Differences**: + +| Feature | PostgreSQL | H2 (PostgreSQL mode) | +|-----------------|-------------------------|-----------------------------| +| UUID default | `gen_random_uuid()` | `random_uuid()` | +| JSON column | `JSONB` | `TEXT` | +| `?::jsonb` cast | Supported | Use plain `TEXT` parameter | +| `ON CONFLICT` | Supported | `MERGE INTO` or `INSERT IGNORE` — H2 2.x supports `ON CONFLICT` in PostgreSQL mode | +| `RETURNING` | Supported | Not supported — use `CALL IDENTITY()` or `getGeneratedKeys()` | + +### 4. Adapt DAOs for Dialect Differences + +The `IndexingRequestDao.create()` method uses `RETURNING id`, which H2 doesn't support. Use JDBC's `getGeneratedKeys()` instead — this works for both databases: + +```java +public UUID create(String player, String platform, String startMonth, String endMonth) { + String sql = "INSERT INTO indexing_requests (id, player, platform, start_month, end_month) VALUES (?, ?, ?, ?, ?)"; + UUID id = UUID.randomUUID(); + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setObject(1, id); + ps.setString(2, player); + ps.setString(3, platform); + ps.setString(4, startMonth); + ps.setString(5, endMonth); + ps.executeUpdate(); + return id; + } +} +``` + +This approach generates the UUID in Java, which works identically on both databases and avoids the `RETURNING` clause entirely. + +The `GameFeatureDao.insert()` method uses `?::jsonb` for the motifs column. In H2 mode, this becomes a plain string parameter. + +```java +public void insert(GameFeatureRow row) { + String sql = isH2 + ? "INSERT INTO game_features (..., motifs_json, ...) VALUES (..., ?, ...) ON CONFLICT (game_url) DO NOTHING" + : "INSERT INTO game_features (..., motifs_json, ...) VALUES (..., ?::jsonb, ...) ON CONFLICT (game_url) DO NOTHING"; + // ... +} +``` + +### 5. IndexerModule Wiring + +```java +@Factory +public class IndexerModule { + + @Context + public DataSource dataSource( + @Value("${indexer.mode:postgres}") String mode, + @Value("${indexer.db.url:jdbc:postgresql://localhost:5432/indexer}") String jdbcUrl, + @Value("${indexer.db.username:indexer}") String username, + @Value("${indexer.db.password:indexer}") String password) { + return switch (mode) { + case "in-process" -> DataSourceFactory.createInMemory(); + default -> DataSourceFactory.create(jdbcUrl, username, password); + }; + } + + // Queue is already InMemoryIndexQueue by default — no change needed +} +``` + +### 6. Configuration + +Environment variables control the mode (H2 in-memory is the default): + +| Variable | Default Value | Effect | +|-----------------------|------------------------------------------|---------------------------------------------| +| `INDEXER_DB_URL` | `jdbc:h2:mem:indexer;DB_CLOSE_DELAY=-1` | H2 in-memory (default). No external deps. | +| `INDEXER_DB_URL` | `jdbc:postgresql://localhost:5432/...` | PostgreSQL mode. Requires external database.| +| `INDEXER_DB_USERNAME` | `sa` | Database username. | +| `INDEXER_DB_PASSWORD` | (empty) | Database password. | + +The system auto-detects H2 vs PostgreSQL from the JDBC URL and uses the appropriate SQL dialect. + +## Operational Characteristics + +### What Works + +- All API endpoints (`POST /index`, `GET /index/{id}`, `POST /query`) +- Full ChessQL query support +- Full motif detection pipeline +- chess.com API fetching (still makes real HTTP calls) +- Concurrent indexing requests via the queue + +### What Doesn't Persist + +- All data is lost when the process exits +- No crash recovery — interrupted indexing jobs are gone +- No way to share data between instances + +### Performance Profile + +| Metric | In-Process (H2) | PostgreSQL | +|-------------------------|------------------|--------------------| +| Insert throughput | ~50K rows/sec | ~5-10K rows/sec | +| Simple query latency | < 1ms | 1-5ms | +| Complex query latency | 1-5ms | 5-50ms | +| Memory per 10K games | ~50MB heap | ~0 (on disk) | +| Max practical dataset | ~100K games | Millions | +| Startup time | ~2s | ~3s (with migration)| + +H2 in-memory is significantly faster for small datasets because there's no network round-trip or disk I/O. The bottleneck shifts entirely to the chess.com API fetch and PGN replay. + +### Memory Sizing + +``` +Base JVM overhead: ~100MB +H2 per 10K games: ~50MB +Chariot replay state: ~20MB (per concurrent replay) +Queue overhead: Negligible + +Recommended heap: + -Xmx512m for < 50K games + -Xmx1g for < 100K games + -Xmx2g for < 200K games (pushing H2 limits) +``` + +Beyond ~200K games, switch to PostgreSQL mode. H2 in-memory keeps the entire dataset in the Java heap, and GC pressure becomes the dominant cost. + +## CLI Mode (Future Extension) + +In-process mode enables a non-HTTP CLI workflow: + +```bash +# Index and query in one shot, no server +java -jar indexer.jar --cli \ + --index hikaru chess.com 2024-03 2024-03 \ + --query "motif(fork) AND white.elo > 2500" \ + --format json +``` + +Implementation: +- Detect `--cli` flag in `App.main()` +- Skip Micronaut server startup +- Wire beans manually (or use Micronaut `ApplicationContext` without HTTP) +- Run indexing synchronously (bypass queue, call `IndexWorker.process()` directly) +- Run query, print results, exit + +This shares 100% of the engine, motif detection, and ChessQL code with the server mode. + +## Testing Benefits + +In-process mode makes integration tests trivial: + +```java +public class IntegrationTest { + private DataSource ds; + private GameFeatureDao dao; + private SqlCompiler compiler; + + @Before + public void setUp() { + ds = DataSourceFactory.createInMemory(); + new Migration(ds).run(); + dao = new GameFeatureDao(ds); + compiler = new SqlCompiler(); + } + + @Test + public void testEndToEndQuery() { + // Insert test data directly + dao.insert(testRow("game1", 2500, 2400, true, false, false, false, false)); + dao.insert(testRow("game2", 2100, 2000, false, true, false, false, false)); + + // Query via ChessQL + Expr expr = Parser.parse("white.elo >= 2500 AND motif(pin)"); + CompiledQuery cq = compiler.compile(expr); + List results = dao.query(cq, 10, 0); + + assertThat(results).hasSize(1); + assertThat(results.get(0).whiteElo()).isEqualTo(2500); + } +} +``` + +No test containers. No Docker. No database setup. Sub-second test execution. + +## Files Modified + +| File | Description | +|------|-------------| +| `bazel/java.MODULE.bazel` | Added `com.h2database:h2:2.2.224` | +| `Migration.java` | H2 dialect detection + compatible DDL (separate SQL for H2 vs PostgreSQL) | +| `GameFeatureDao.java` | Dialect-aware `jsonb` cast and `MERGE` vs `ON CONFLICT` | +| `IndexerModule.java` | Default to H2 in-memory, auto-detect dialect from JDBC URL | +| `db/BUILD.bazel` | Added H2 runtime dep | + +The entire in-process mode is a configuration default with ~100 lines of dialect adaptation across existing files. + +## Comparison with Alternatives + +| Approach | Startup | External Deps | SQL Compat | JSONB | Effort | +|--------------------|---------|---------------|------------|-------|--------| +| **H2 in-memory** | ~2s | None | High | Partial (TEXT fallback) | Low | +| SQLite (via JDBC) | ~2s | Native lib | Medium | No | Medium | +| HSQLDB in-memory | ~2s | None | Medium | No | Medium | +| Pure Java maps | ~1s | None | None | N/A | High (rewrite DAOs) | +| Testcontainers PG | ~10s | Docker | Perfect | Yes | Low | + +H2 is the best tradeoff: pure Java (no native libs, works in Bazel sandbox), high PostgreSQL compatibility, minimal code changes, and sub-second database startup. diff --git a/jvm/src/main/java/com/muchq/indexer/docs/MCP_INTEGRATION.md b/jvm/src/main/java/com/muchq/indexer/docs/MCP_INTEGRATION.md new file mode 100644 index 00000000..052c5ce5 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/docs/MCP_INTEGRATION.md @@ -0,0 +1,495 @@ +# Chess Game Indexer — MCP Server Integration + +## Overview + +Expose the indexer's capabilities as MCP (Model Context Protocol) tool calls, so that LLM agents connected to the existing `mcpserver` can index chess games and query them using ChessQL — all through natural language. + +## Architecture + +There are two integration approaches. Both are viable; choose based on deployment topology. + +### Option A: In-Process (Recommended) + +Add indexer tool classes directly to the `mcpserver` package. The mcpserver process embeds the indexer engine, DB access, and queue. Uses in-process mode (H2) by default with PostgreSQL as an option. + +``` +┌──────────────────────────────────────────────────┐ +│ mcpserver JVM Process │ +│ │ +│ McpRequestHandler │ +│ └─ ToolRegistry │ +│ ├─ ChessComGamesTool (existing) │ +│ ├─ ChessComPlayerTool (existing) │ +│ ├─ ChessComStatsTool (existing) │ +│ ├─ ServerTimeTool (existing) │ +│ ├─ IndexGamesTool (NEW) │ +│ ├─ IndexStatusTool (NEW) │ +│ ├─ QueryGamesTool (NEW) │ +│ └─ AnalyzePositionTool (NEW) │ +│ │ │ +│ ┌────────▼─────────┐ │ +│ │ IndexerFacade │ │ +│ │ (thin wrapper) │ │ +│ └────────┬─────────┘ │ +│ │ │ +│ ┌─────────────▼──────────────────┐ │ +│ │ Indexer Engine + H2/Postgres │ │ +│ └────────────────────────────────┘ │ +└──────────────────────────────────────────────────┘ +``` + +**Pros**: Single process, no network hops, in-process mode works out of the box, simplest deployment. +**Cons**: Larger binary, indexer lifecycle coupled to mcpserver. + +### Option B: HTTP Proxy + +MCP tools call the indexer's REST API over HTTP. The indexer runs as a separate process. + +``` +┌────────────────────┐ HTTP ┌──────────────────┐ +│ mcpserver │ ────────────────► │ indexer │ +│ IndexGamesTool ───┤ POST /index │ IndexController │ +│ QueryGamesTool ───┤ POST /query │ QueryController │ +└────────────────────┘ └──────────────────┘ +``` + +**Pros**: Independent scaling, separate deploys, clear service boundary. +**Cons**: Network latency, two processes to manage, need indexer running. + +--- + +## Tool Definitions + +### 1. `index_chess_games` + +Start indexing a player's games for motif detection. + +**Input Schema**: +```json +{ + "type": "object", + "properties": { + "username": { + "type": "string", + "description": "The player's username on the chess platform" + }, + "platform": { + "type": "string", + "enum": ["chess.com"], + "description": "The chess platform (currently only chess.com)" + }, + "start_month": { + "type": "string", + "description": "Start month in YYYY-MM format (e.g. 2024-03)" + }, + "end_month": { + "type": "string", + "description": "End month in YYYY-MM format (e.g. 2024-03)" + } + }, + "required": ["username", "platform", "start_month", "end_month"] +} +``` + +**Output**: JSON with request ID and initial status. + +``` +{"id": "abc-123", "status": "PENDING", "gamesIndexed": 0} +``` + +**Example LLM Interaction**: +> User: "Index hikaru's games from March 2024" +> LLM calls `index_chess_games(username="hikaru", platform="chess.com", start_month="2024-03", end_month="2024-03")` + +### 2. `index_status` + +Check the status of an indexing request. + +**Input Schema**: +```json +{ + "type": "object", + "properties": { + "request_id": { + "type": "string", + "description": "The UUID of the indexing request" + } + }, + "required": ["request_id"] +} +``` + +**Output**: JSON with current status, game count, and any error. + +``` +{"id": "abc-123", "status": "COMPLETED", "gamesIndexed": 147, "errorMessage": null} +``` + +**Example LLM Interaction**: +> User: "Is the indexing done yet?" +> LLM calls `index_status(request_id="abc-123")` +> LLM: "Yes, indexing is complete. 147 games were indexed." + +### 3. `query_chess_games` + +Search indexed games using ChessQL. + +**Input Schema**: +```json +{ + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "A ChessQL query string. Examples: 'white.elo >= 2500 AND motif(fork)', 'motif(pin) OR motif(skewer)', 'eco = \"B90\" AND NOT motif(fork)'. Available motifs: pin, cross_pin, fork, skewer, discovered_attack. Available fields: white.elo, black.elo, white.username, black.username, time.class, eco, result, num.moves, platform." + }, + "limit": { + "type": "integer", + "description": "Maximum number of results to return (default 10, max 50)" + } + }, + "required": ["query"] +} +``` + +**Output**: JSON array of matching games with key fields. + +``` +{"games": [{"gameUrl": "...", "whiteUsername": "hikaru", "whiteElo": 2850, ...}], "count": 3} +``` + +**Example LLM Interactions**: +> User: "Find hikaru's games where he played a fork as white" +> LLM calls `query_chess_games(query="white.username = \"hikaru\" AND motif(fork)", limit=10)` + +> User: "Show me games with both pins and forks where someone was rated over 2500" +> LLM calls `query_chess_games(query="motif(pin) AND motif(fork) AND (white.elo >= 2500 OR black.elo >= 2500)")` + +> User: "Any Sicilian Najdorf games with discovered attacks?" +> LLM calls `query_chess_games(query="eco = \"B90\" AND motif(discovered_attack)")` + +### 4. `analyze_position` + +Detect motifs in a single PGN without indexing it to the database. + +**Input Schema**: +```json +{ + "type": "object", + "properties": { + "pgn": { + "type": "string", + "description": "A PGN string of the chess game to analyze" + } + }, + "required": ["pgn"] +} +``` + +**Output**: JSON with detected motifs and occurrence details. + +``` +{ + "numMoves": 42, + "motifs": ["PIN", "FORK"], + "occurrences": { + "PIN": [{"moveNumber": 15, "description": "Pin detected at move 15"}], + "FORK": [{"moveNumber": 23, "description": "Fork detected at move 23"}] + } +} +``` + +**Example LLM Interaction**: +> User: "Analyze this game for tactics: [Event \"Live\"] ... 1. e4 e5 ..." +> LLM calls `analyze_position(pgn="[Event \"Live\"] ...")` +> LLM: "I found two tactical motifs: a pin at move 15 and a knight fork at move 23." + +--- + +## Implementation: Option A (In-Process) + +### New Files + +All under `jvm/src/main/java/com/muchq/mcpserver/tools/`: + +#### IndexerFacade.java + +Thin wrapper that manages indexer components without exposing internal types to MCP tools: + +```java +public class IndexerFacade { + private final IndexingRequestDao requestDao; + private final GameFeatureDao gameFeatureDao; + private final IndexQueue queue; + private final IndexWorker worker; + private final FeatureExtractor featureExtractor; + private final SqlCompiler sqlCompiler; + + // Synchronous index (blocks until complete — suitable for small ranges) + public IndexResult indexSync(String player, String platform, String startMonth, String endMonth) { ... } + + // Async index (enqueues and returns immediately) + public IndexResult indexAsync(String player, String platform, String startMonth, String endMonth) { ... } + + // Poll status + public IndexResult getStatus(UUID requestId) { ... } + + // Query + public QueryResult query(String chessql, int limit) { ... } + + // Analyze a single PGN + public AnalysisResult analyze(String pgn) { ... } +} +``` + +#### IndexGamesTool.java + +```java +public class IndexGamesTool implements McpTool { + private final IndexerFacade facade; + private final ObjectMapper mapper; + + @Override + public String getName() { return "index_chess_games"; } + + @Override + public String getDescription() { + return "Index a chess player's games for tactical motif detection. " + + "Fetches games from chess.com, replays positions, and detects " + + "pins, forks, skewers, discovered attacks, and cross-pins. " + + "Returns a request ID to check status with index_status."; + } + + @Override + public Map getInputSchema() { + return Map.of( + "type", "object", + "properties", Map.of( + "username", Map.of("type", "string", "description", "Chess platform username"), + "platform", Map.of("type", "string", "enum", List.of("chess.com"), + "description", "Chess platform"), + "start_month", Map.of("type", "string", + "description", "Start month (YYYY-MM format)"), + "end_month", Map.of("type", "string", + "description", "End month (YYYY-MM format)") + ), + "required", List.of("username", "platform", "start_month", "end_month") + ); + } + + @Override + public String execute(Map arguments) { + String username = (String) arguments.get("username"); + String platform = (String) arguments.get("platform"); + String startMonth = (String) arguments.get("start_month"); + String endMonth = (String) arguments.get("end_month"); + + var result = facade.indexAsync(username, platform, startMonth, endMonth); + return serialize(result); + } +} +``` + +#### IndexStatusTool.java, QueryGamesTool.java, AnalyzePositionTool.java + +Same pattern as above — implement `McpTool`, delegate to `IndexerFacade`. + +### McpModule Changes + +Wire the new tools into the tool registry: + +```java +@Context +public List mcpTools( + Clock clock, + ChessClient chessClient, + ObjectMapper objectMapper, + IndexerFacade indexerFacade) { + return List.of( + new ChessComGamesTool(chessClient, objectMapper), + new ChessComPlayerTool(chessClient, objectMapper), + new ChessComStatsTool(chessClient, objectMapper), + new ServerTimeTool(clock), + new IndexGamesTool(indexerFacade, objectMapper), // NEW + new IndexStatusTool(indexerFacade, objectMapper), // NEW + new QueryGamesTool(indexerFacade, objectMapper), // NEW + new AnalyzePositionTool(indexerFacade, objectMapper) // NEW + ); +} + +@Context +public IndexerFacade indexerFacade(...) { + // Wire indexer components — reuses existing engine, DB, queue code +} +``` + +### BUILD.bazel Changes + +Add indexer libraries as deps to `mcpserver/tools/BUILD.bazel`: + +```bzl +deps = [ + # existing deps... + "//jvm/src/main/java/com/muchq/indexer/chessql/compiler", + "//jvm/src/main/java/com/muchq/indexer/chessql/parser", + "//jvm/src/main/java/com/muchq/indexer/db", + "//jvm/src/main/java/com/muchq/indexer/engine", + "//jvm/src/main/java/com/muchq/indexer/engine/model", + "//jvm/src/main/java/com/muchq/indexer/queue", + "//jvm/src/main/java/com/muchq/indexer/worker", +], +``` + +And H2 as a runtime dep for in-process mode on `mcpserver/BUILD.bazel`. + +--- + +## Implementation: Option B (HTTP Proxy) + +### New Files + +Same tool classes, but `IndexerFacade` calls the indexer REST API instead: + +```java +public class IndexerHttpClient { + private final HttpClient httpClient; + private final String baseUrl; // e.g. http://localhost:8080 + + public IndexResult index(String player, String platform, String startMonth, String endMonth) { + // POST http://localhost:8080/index + // Body: {"player":"...","platform":"...","startMonth":"...","endMonth":"..."} + } + + public IndexResult getStatus(UUID requestId) { + // GET http://localhost:8080/index/{id} + } + + public QueryResult query(String chessql, int limit) { + // POST http://localhost:8080/query + // Body: {"query":"...","limit":N,"offset":0} + } +} +``` + +This reuses the existing `HttpClient` abstraction (`com.muchq.http_client.core`). + +No `analyze_position` over HTTP unless we add a dedicated endpoint to the indexer API. Alternatively, embed just the engine (no DB) in the mcpserver for analysis. + +--- + +## Sync vs Async Indexing in MCP Context + +MCP tool calls are synchronous — the LLM waits for a response. Indexing a month of games can involve hundreds of API calls to chess.com. + +### Strategy: Hybrid + +1. **Small requests (1 month)**: Run synchronously. The tool blocks until indexing is complete. Most months have < 500 games, completing in 30-60 seconds. The LLM can present results immediately. + +2. **Large requests (multi-month)**: Run asynchronously. Return the request ID immediately. The LLM then polls with `index_status` in a follow-up turn. + +```java +public String execute(Map arguments) { + // ... + YearMonth start = YearMonth.parse(startMonth); + YearMonth end = YearMonth.parse(endMonth); + long monthSpan = start.until(end, java.time.temporal.ChronoUnit.MONTHS) + 1; + + if (monthSpan <= 1) { + // Synchronous — block and return full result + var result = facade.indexSync(username, platform, startMonth, endMonth); + return serialize(result); + } else { + // Async — enqueue and return ID + var result = facade.indexAsync(username, platform, startMonth, endMonth); + return serialize(result) + "\n\nNote: Multi-month indexing runs in the background. " + + "Use index_status to check progress."; + } +} +``` + +--- + +## Conversation Flow Examples + +### Example 1: Index and Query + +``` +User: Index hikaru's blitz games from January 2024 and find games with forks + +LLM: I'll index hikaru's games first, then search for forks. + +[Tool call: index_chess_games(username="hikaru", platform="chess.com", + start_month="2024-01", end_month="2024-01")] +→ {"id": "abc-123", "status": "COMPLETED", "gamesIndexed": 312} + +[Tool call: query_chess_games(query="white.username = \"hikaru\" AND motif(fork)", + limit=10)] +→ {"games": [...], "count": 7} + +LLM: I indexed 312 of hikaru's games from January 2024. I found 7 games where + hikaru played as white and executed a fork. Here are the highlights... +``` + +### Example 2: Analyze a Specific Game + +``` +User: Can you analyze this game for tactics? + 1. e4 e5 2. Nf3 Nc6 3. Bb5 a6 4. Ba4 Nf6 5. O-O ... + +[Tool call: analyze_position(pgn="1. e4 e5 2. Nf3 Nc6 ...")] +→ {"numMoves": 35, "motifs": ["PIN", "FORK"], + "occurrences": {"PIN": [{"moveNumber": 12, ...}], "FORK": [{"moveNumber": 23, ...}]}} + +LLM: I found two tactical motifs in this game: + - A pin at move 12 where... + - A knight fork at move 23 where... +``` + +### Example 3: Complex Research Query + +``` +User: Are there more forks or skewers in high-elo Sicilian games? + +LLM: Let me search the indexed games for both patterns. + +[Tool call: query_chess_games(query="eco IN [\"B20\",\"B90\",\"B91\",\"B92\"] + AND (white.elo >= 2500 OR black.elo >= 2500) AND motif(fork)", limit=50)] +→ {"games": [...], "count": 23} + +[Tool call: query_chess_games(query="eco IN [\"B20\",\"B90\",\"B91\",\"B92\"] + AND (white.elo >= 2500 OR black.elo >= 2500) AND motif(skewer)", limit=50)] +→ {"games": [...], "count": 8} + +LLM: In the indexed high-elo Sicilian games, forks are nearly 3x more common + than skewers (23 vs 8 games). This makes sense because... +``` + +--- + +## Files Summary + +### Option A (In-Process) + +| File | Action | Description | +|------|--------|-------------| +| `mcpserver/tools/IndexerFacade.java` | Create | Thin wrapper over indexer components | +| `mcpserver/tools/IndexGamesTool.java` | Create | `index_chess_games` MCP tool | +| `mcpserver/tools/IndexStatusTool.java` | Create | `index_status` MCP tool | +| `mcpserver/tools/QueryGamesTool.java` | Create | `query_chess_games` MCP tool | +| `mcpserver/tools/AnalyzePositionTool.java` | Create | `analyze_position` MCP tool | +| `mcpserver/tools/BUILD.bazel` | Modify | Add indexer library deps | +| `mcpserver/McpModule.java` | Modify | Wire IndexerFacade + new tools | +| `mcpserver/BUILD.bazel` | Modify | Add indexer + H2 deps | + +### Option B (HTTP Proxy) + +| File | Action | Description | +|------|--------|-------------| +| `mcpserver/tools/IndexerHttpClient.java` | Create | HTTP client to indexer API | +| `mcpserver/tools/IndexGamesTool.java` | Create | Delegates to IndexerHttpClient | +| `mcpserver/tools/IndexStatusTool.java` | Create | Delegates to IndexerHttpClient | +| `mcpserver/tools/QueryGamesTool.java` | Create | Delegates to IndexerHttpClient | +| `mcpserver/tools/BUILD.bazel` | Modify | Add http_client dep | +| `mcpserver/McpModule.java` | Modify | Wire IndexerHttpClient + new tools | + +Option A is recommended for initial implementation. It requires no separate process, works with in-process mode, and can be split into Option B later if scaling demands it. diff --git a/jvm/src/main/java/com/muchq/indexer/docs/ROADMAP.md b/jvm/src/main/java/com/muchq/indexer/docs/ROADMAP.md new file mode 100644 index 00000000..1326f3ea --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/docs/ROADMAP.md @@ -0,0 +1,593 @@ +# Chess Game Indexer — Roadmap + +## Current State (Phase 1 — Delivered) + +- Indexing pipeline: chess.com → PGN parse → position replay → motif detection → PostgreSQL +- ChessQL query language with parameterized SQL compilation +- In-memory queue with interface abstraction +- 5 motif detectors: pin, cross-pin, fork, skewer, discovered attack +- Bazel build with OCI image target +- Test coverage for ChessQL (lexer, parser, compiler), PGN parser, queue + +### Known Gaps + +- No input validation on API DTOs +- No structured error responses (exceptions propagate as 500s) +- No authentication or rate limiting +- Single worker thread, no concurrency control +- No retry logic for chess.com API failures +- No data retention or lifecycle management +- No observability beyond SLF4J logging +- Motif occurrences only record move number, not the actual move notation (see below) + +### Motif Recording Enhancement + +Currently, `MotifOccurrence` records only the move number where a motif was detected: +```json +{"moveNumber": 12, "description": "Fork detected at move 12"} +``` + +This should be enhanced to record the actual move in algebraic notation: +- White knight fork on move 12: `"12. Nf3"` (or the square the knight moved to) +- Black bishop fork after capturing on d5: `"14...Bxd5"` +- Include the piece type, source/destination squares, and capture notation + +**Implementation notes:** +- `PositionContext` needs to include the move that led to the position (currently only has FEN and move number) +- `GameReplayer` should pass the SAN (Standard Algebraic Notation) for each move +- `MotifOccurrence` should store: `moveNumber`, `san` (e.g., "Nf3"), `fullNotation` (e.g., "12. Nf3"), `piece`, `fromSquare`, `toSquare` + +This enables richer query results and makes it possible to jump directly to the tactical moment in a game viewer. + +--- + +## Phase 2 — Validation & Error Handling + +### Input Validation + +Add Micronaut validation annotations to DTOs: + +```java +public record IndexRequest( + @NotBlank String player, + @NotBlank @Pattern(regexp = "chess\\.com|lichess") String platform, + @NotBlank @Pattern(regexp = "\\d{4}-\\d{2}") String startMonth, + @NotBlank @Pattern(regexp = "\\d{4}-\\d{2}") String endMonth +) {} +``` + +Additional validations: +- `startMonth <= endMonth` (semantic check in controller) +- Month range cap (max 12 months per request to bound work) +- `QueryRequest.query` max length (prevent abuse) + +### Error Mapping + +Add a Micronaut `@Error` handler or exception mapper: + +| Exception | HTTP Status | Response Body | +|-------------------------|-------------|--------------------------------------| +| `ParseException` | 400 | `{"error": "...", "position": N}` | +| `IllegalArgumentException` | 400 | `{"error": "..."}` | +| Entity not found | 404 | `{"error": "Not found"}` | +| Unexpected | 500 | `{"error": "Internal error"}` | + +### Duplicate Request Detection + +Before creating a new indexing request, check if an identical (player, platform, startMonth, endMonth) request already exists with status PENDING or PROCESSING. Return the existing request ID instead of creating a duplicate. + +### Historical Period Caching + +Avoid re-fetching games for player/platform/month combinations that have already been fully indexed. + +**Key insight:** A month's games are only "complete" if the fetch occurred *after* the month ended. For example: +- Fetching hikaru's January 2024 games on February 1, 2024 → complete, safe to cache +- Fetching hikaru's January 2024 games on January 15, 2024 → partial, should re-fetch later + +**Implementation:** + +1. New table to track fetched periods: +```sql +CREATE TABLE indexed_periods ( + id UUID PRIMARY KEY, + player VARCHAR(255) NOT NULL, + platform VARCHAR(50) NOT NULL, + month VARCHAR(7) NOT NULL, -- "2024-01" + fetched_at TIMESTAMP NOT NULL, + is_complete BOOLEAN NOT NULL, -- true if fetched_at > end of month + games_count INT NOT NULL, + UNIQUE (player, platform, month) +); +``` + +2. Before fetching a month's games: + - Check if `indexed_periods` has a complete entry for (player, platform, month) + - If complete: skip fetch, return existing game count + - If incomplete or missing: fetch from API, upsert the period record + +3. Mark `is_complete = true` only if `fetched_at > last day of month` + +4. Optional: Admin endpoint to invalidate cached periods and force re-fetch + +**Edge cases:** +- Player changes username → old username's cache is stale (detect via player ID if API provides it) +- Games added retroactively by chess.com (rare) → accept minor staleness or add TTL + +### Estimated Changes + +- 3-4 files modified (DTOs, controllers) +- 1-2 new files (error handler, validation config) +- ~200 lines of code + +--- + +## Phase 3 — Resilience & Retry + +### Chess.com API Resilience + +The `ChessClient` currently throws on non-200/404 responses. Add: + +**Rate Limiting** +- chess.com API has undocumented rate limits (empirically ~10 req/s) +- Add a `RateLimiter` (token bucket, 5 req/s with burst of 10) +- Implementation: `java.util.concurrent.Semaphore` or Guava `RateLimiter` + +**Retry with Exponential Backoff** +- Retry on HTTP 429 (Too Many Requests) and 5xx errors +- 3 retries, backoff: 1s → 2s → 4s, with jitter +- Implementation: simple retry loop in `ChessClient`, no external library needed + +**Circuit Breaker** +- If consecutive failures exceed threshold (e.g., 5), open circuit for 60s +- Prevents hammering a down API and speeds up failure detection +- Implementation: state machine in a `CircuitBreaker` utility class + +```java +public class CircuitBreaker { + enum State { CLOSED, OPEN, HALF_OPEN } + // Track failures, state transitions, cooldown timer +} +``` + +### Queue Retry + +**Current**: If `IndexWorker.process()` fails, the message is lost (already dequeued). + +**Phase 3 Changes**: +- Add `requeue(IndexMessage message, int attempt)` to `IndexQueue` +- On failure, requeue with incremented attempt count, up to `MAX_ATTEMPTS=3` +- After max attempts, mark request as FAILED with error details +- Add `attempt` field to `IndexMessage` + +**Dead Letter Queue (DLQ)**: +- Messages that exceed max attempts go to a DLQ +- DLQ is a separate `LinkedBlockingQueue` (or SQS DLQ later) +- Admin endpoint `GET /admin/dlq` to inspect failed messages +- Admin endpoint `POST /admin/dlq/{id}/retry` to reprocess + +### Per-Game Error Isolation + +Currently a single game failure is caught and logged, but the overall request continues. Strengthen this: +- Track per-game errors in a `List` on the request +- Store partial results even on overall failure +- Add `games_failed` counter to `indexing_requests` + +### Estimated Changes + +- 4-5 files modified (ChessClient, IndexWorker, IndexQueue, IndexMessage) +- 2-3 new files (RateLimiter, CircuitBreaker, retry utilities) +- ~400 lines of code + +--- + +## Phase 4 — SQS Queue Migration + +### Motivation + +The `InMemoryIndexQueue` loses messages on process restart. SQS provides: +- Durability (messages survive restarts) +- Visibility timeout (automatic redelivery on consumer failure) +- Built-in DLQ support +- Multi-consumer scaling + +### Implementation + +New class `SqsIndexQueue implements IndexQueue`: + +```java +public class SqsIndexQueue implements IndexQueue { + // enqueue → sqs.sendMessage(queueUrl, serialize(message)) + // poll → sqs.receiveMessage(queueUrl, waitTimeSeconds) + // + sqs.deleteMessage(receiptHandle) on success + // size → sqs.getQueueAttributes(ApproximateNumberOfMessages) +} +``` + +**Configuration**: +- `INDEXER_QUEUE_TYPE=memory|sqs` (default: memory) +- `INDEXER_SQS_QUEUE_URL` for SQS mode +- `INDEXER_SQS_DLQ_URL` for dead letter queue +- Visibility timeout: 300s (5 minutes, covers most indexing jobs) +- Max receive count: 3 (then routes to DLQ) + +**IndexerModule Change**: +```java +@Context +public IndexQueue indexQueue(@Value("${indexer.queue.type:memory}") String type, ...) { + return switch (type) { + case "sqs" -> new SqsIndexQueue(sqsClient, queueUrl); + default -> new InMemoryIndexQueue(); + }; +} +``` + +### Dependencies + +- `software.amazon.awssdk:sqs:2.x` added to `java.MODULE.bazel` +- AWS credentials via environment or IAM role + +### Estimated Changes + +- 1 file modified (IndexerModule, java.MODULE.bazel) +- 1-2 new files (SqsIndexQueue, SQS config) +- ~200 lines of code + +--- + +## Phase 5 — Security + +### Authentication + +Add API key authentication via a Micronaut `HttpServerFilter`: + +```java +@Filter("/**") +public class ApiKeyFilter implements HttpServerFilter { + // Check X-API-Key header against configured keys + // 401 if missing, 403 if invalid + // Exempt /health endpoint +} +``` + +**Configuration**: `INDEXER_API_KEYS=key1,key2,key3` + +### Authorization + +Phase 1: Single-tier (any valid key has full access). +Phase 2: Role-based — `admin` keys can access /admin/*, `user` keys can access /index and /query. + +### Rate Limiting + +Per-key rate limiting on the /query endpoint: +- 100 queries/minute per API key +- 429 response with `Retry-After` header +- In-memory counter with sliding window (Guava `Cache`) +- Later: Redis-backed for multi-instance + +### ChessQL Security + +Already addressed: +- All values are parameterized (`?` placeholders) +- Field and motif names validated against whitelists +- No raw string interpolation + +Additional hardening: +- Max query string length (4KB) +- Max AST depth (20 levels) to prevent stack overflow on deeply nested queries +- Query timeout at the DB level (`SET statement_timeout = '5s'`) + +### Estimated Changes + +- 2-3 new files (ApiKeyFilter, rate limiter) +- 1-2 files modified (QueryController for timeout, application.yml for config) +- ~250 lines of code + +--- + +## Phase 6 — Data Retention & Cold Storage + +### Problem + +Game data grows linearly with indexing requests. Without lifecycle management: +- A single player-month can produce 500-2000 games +- Each game row is ~2-5KB (with PGN) +- 1M games ≈ 2-5GB in PostgreSQL +- Query performance degrades as table grows +- Storage costs grow unbounded + +### Retention Policy + +#### Tier 1: Hot Storage (PostgreSQL, 0-30 days) + +All recently indexed games live in the main `game_features` table. Queries hit this table directly. + +- Retention period: 30 days from `played_at` or `created_at` of the indexing request +- Full query capability via ChessQL +- Indexed boolean columns for fast motif queries + +#### Tier 2: Warm Storage (PostgreSQL archive partition, 30-90 days) + +Games older than 30 days are moved to a partitioned archive table with reduced indexing. + +```sql +CREATE TABLE game_features_archive ( + LIKE game_features INCLUDING ALL +) PARTITION BY RANGE (played_at); + +-- Monthly partitions +CREATE TABLE game_features_archive_2024_01 + PARTITION OF game_features_archive + FOR VALUES FROM ('2024-01-01') TO ('2024-02-01'); +``` + +- Queries can optionally span both tables (`include_archive=true` on QueryRequest) +- Reduced indexes (drop individual motif indexes, keep composite) +- PGN column retained for re-analysis + +#### Tier 3: Cold Storage (S3/GCS, 90+ days) + +Games older than 90 days are exported to object storage and deleted from PostgreSQL. + +**Export Format**: Newline-delimited JSON (NDJSON), gzipped, partitioned by month: +``` +s3://indexer-archive/games/2024/01/games-2024-01.ndjson.gz +s3://indexer-archive/games/2024/02/games-2024-02.ndjson.gz +``` + +- No direct query capability — must re-index to search +- Admin endpoint `POST /admin/restore?month=2024-01` to re-import from cold storage +- PGN preserved for full re-analysis with updated detectors + +#### Tier 4: Deletion (365+ days) + +Cold storage objects older than 1 year are deleted via S3 lifecycle policy. This is configurable per deployment. + +### Implementation: Retention Worker + +New `RetentionWorker` daemon (similar to `IndexWorkerLifecycle`), runs daily: + +```java +public class RetentionWorker { + // 1. Move hot → warm: INSERT INTO game_features_archive SELECT ... WHERE played_at < now() - interval '30 days' + // DELETE FROM game_features WHERE played_at < now() - interval '30 days' + // 2. Move warm → cold: Export to S3, DROP partition + // 3. Log metrics: rows moved, bytes exported, partitions dropped +} +``` + +**Configuration**: + +| Variable | Default | Description | +|-----------------------------------|---------|---------------------------------| +| `INDEXER_RETENTION_HOT_DAYS` | 30 | Days in hot storage | +| `INDEXER_RETENTION_WARM_DAYS` | 90 | Days in warm storage | +| `INDEXER_RETENTION_COLD_DAYS` | 365 | Days in cold storage | +| `INDEXER_ARCHIVE_BUCKET` | — | S3/GCS bucket for cold storage | +| `INDEXER_RETENTION_ENABLED` | false | Master switch | +| `INDEXER_RETENTION_CRON` | 0 3 * * * | Daily at 3 AM | + +### Migration Path + +The retention system can be introduced incrementally: +1. Add `played_at` index to `game_features` (already exists as column) +2. Add archive table and partition scheme +3. Add retention worker for hot→warm +4. Add S3 export for warm→cold +5. Add restore endpoint + +### Estimated Changes + +- 3-4 new files (RetentionWorker, S3Archiver, archive migration, retention config) +- 2-3 files modified (Migration.java for archive table, QueryController for archive queries) +- ~500 lines of code + +--- + +## Phase 7 — Observability + +### Structured Logging + +Replace ad-hoc log messages with structured key-value logging: + +```java +LOG.info("index.game.processed", kv("requestId", id), kv("gameUrl", url), kv("motifs", count)); +``` + +Use Logback's `LogstashEncoder` for JSON log output in production. + +### Metrics + +Expose Micrometer metrics via Micronaut's built-in support: + +| Metric | Type | Description | +|---------------------------------|-----------|------------------------------------| +| `indexer.requests.created` | Counter | Total indexing requests created | +| `indexer.requests.completed` | Counter | Successfully completed | +| `indexer.requests.failed` | Counter | Failed requests | +| `indexer.games.indexed` | Counter | Total games indexed | +| `indexer.games.motifs.detected` | Counter | Motifs found (tagged by motif type) | +| `indexer.queue.size` | Gauge | Current queue depth | +| `indexer.query.duration` | Timer | ChessQL query execution time | +| `indexer.chesscom.requests` | Counter | API calls to chess.com | +| `indexer.chesscom.errors` | Counter | API errors (tagged by status code) | +| `indexer.retention.moved` | Counter | Rows moved per tier transition | + +### Health Checks + +``` +GET /health → {"status": "UP", "checks": {...}} +GET /health/liveness → 200 if process is alive +GET /health/readiness → 200 if DB is reachable and queue is functional +``` + +### Estimated Changes + +- 2-3 new files (health check, metrics config) +- 5-6 files modified (add metrics to worker, controllers, ChessClient) +- ~300 lines of code + +--- + +## Phase 8 — Lichess Support + +### Motivation + +The platform abstraction (`platform` column, `IndexMessage.platform`) was designed for multi-platform support from the start. + +### Implementation + +New `LichessClient` alongside `ChessClient`: +- Lichess API: `https://lichess.org/api/games/user/{username}?since=...&until=...` +- Returns PGN stream (not JSON) — needs streaming parser +- Rate limit: 20 req/s (more generous than chess.com) + +**Platform Router**: +```java +public class GameFetcher { + public List fetch(String player, String platform, YearMonth month) { + return switch (platform) { + case "chess.com" -> chessComClient.fetchGames(player, month); + case "lichess" -> lichessClient.fetchGames(player, month); + default -> throw new IllegalArgumentException("Unknown platform: " + platform); + }; + } +} +``` + +### Dependencies + +- `io.github.tors42:chariot` already supports Lichess API — evaluate using it directly +- Alternatively, use raw HTTP client for the PGN stream endpoint + +### Estimated Changes + +- 2-3 new files (LichessClient, GameFetcher) +- 2-3 files modified (IndexWorker, IndexerModule) +- ~300 lines of code + +--- + +## Phase 9 — Additional Motifs & Re-Analysis + +### New Motifs + +| Motif | Detection Strategy | +|----------------------|------------------------------------------------------| +| Back rank mate | Checkmate with king on 1st/8th rank, blocked by pawns | +| Smothered mate | Knight checkmate, king surrounded by own pieces | +| Sacrifice | Piece captured where capturer is higher value | +| Zugzwang | Position where any move worsens the position (heuristic) | +| Double check | Two pieces give check simultaneously | +| Interference | Piece placed to block an enemy piece's line | +| Overloaded piece | Piece defending two or more targets simultaneously | + +### Re-Analysis Pipeline + +When new motif detectors are added, existing games need re-analysis: + +1. Admin endpoint: `POST /admin/reanalyze?motif=back_rank_mate` +2. Reads PGN from `game_features.pgn` column +3. Replays and runs only the new detector +4. Updates boolean column and `motifs_json` +5. Batched processing (1000 games per batch) to avoid memory pressure + +### Schema Evolution + +Adding a new motif column: +```sql +ALTER TABLE game_features ADD COLUMN has_back_rank_mate BOOLEAN DEFAULT FALSE; +ALTER TABLE game_features_archive ADD COLUMN has_back_rank_mate BOOLEAN DEFAULT FALSE; +``` + +Update `SqlCompiler.VALID_MOTIFS` and `VALID_COLUMNS` sets. + +--- + +## Cost & Ops Complexity Estimates + +### Infrastructure Costs (AWS, us-east-1, monthly) + +#### Small Deployment (< 100K games) + +| Resource | Spec | Estimated Cost | +|-----------------------------|-----------------------------|----------------| +| PostgreSQL (RDS) | db.t4g.micro, 20GB gp3 | $15-20 | +| EC2 / ECS (app) | t4g.small (2 vCPU, 2GB) | $12-15 | +| S3 (cold storage) | < 1GB | < $1 | +| SQS | < 1M messages/month | < $1 | +| **Total** | | **~$30/month** | + +Ops complexity: **Low**. Single instance, automated backups, no scaling concerns. Can run on a single t4g.small with embedded PostgreSQL for development. + +#### Medium Deployment (100K - 1M games) + +| Resource | Spec | Estimated Cost | +|-----------------------------|-----------------------------|----------------| +| PostgreSQL (RDS) | db.t4g.medium, 100GB gp3 | $50-70 | +| EC2 / ECS (app, 2x) | t4g.medium (2 vCPU, 4GB) | $50-60 | +| S3 (cold storage) | 10-50GB | $1-2 | +| SQS | < 10M messages/month | < $5 | +| CloudWatch / monitoring | Basic | $10-15 | +| **Total** | | **~$120-150/month** | + +Ops complexity: **Medium**. Need connection pooling tuning, query performance monitoring, retention job scheduling. Recommend adding an ALB ($16/month) for health checks and graceful deploys. Consider read replicas if query load is high. + +#### Large Deployment (1M+ games) + +| Resource | Spec | Estimated Cost | +|-----------------------------|-----------------------------|-----------------| +| PostgreSQL (RDS) | db.r7g.large, 500GB gp3, read replica | $300-400 | +| ECS Fargate (app, 3x) | 1 vCPU, 2GB | $80-100 | +| ECS Fargate (workers, 2x) | 1 vCPU, 2GB | $50-70 | +| S3 (cold storage) | 50-500GB | $5-15 | +| SQS + DLQ | Standard | $5-10 | +| ALB | Standard | $20-25 | +| CloudWatch + alarms | Enhanced | $30-50 | +| **Total** | | **~$500-650/month** | + +Ops complexity: **High**. Partition management for archive tables, retention job monitoring, S3 lifecycle policies, multi-worker coordination (SQS visibility timeout tuning), query performance (may need `EXPLAIN ANALYZE` review, index tuning). Consider: +- Splitting reads and writes to separate DB instances +- Connection pooling via PgBouncer +- Caching frequent ChessQL queries (Redis, ~$15/month for cache.t4g.micro) + +### Storage Growth Model + +| Metric | Per Game | 100K Games | 1M Games | +|----------------------------|----------|------------|-----------| +| game_features row (no PGN) | ~500B | ~50MB | ~500MB | +| PGN column | ~2-3KB | ~250MB | ~2.5GB | +| motifs_json column | ~200B | ~20MB | ~200MB | +| **Total per game** | ~3KB | ~300MB | ~3GB | +| Indexes overhead | ~30% | ~100MB | ~1GB | +| **Total with indexes** | | ~400MB | ~4GB | + +Cold storage (gzipped NDJSON) achieves ~5:1 compression, so 1M games ≈ 600MB in S3. + +### Ops Complexity Summary + +| Phase | Complexity | New Ops Burden | +|-------|------------|--------------------------------------------------------| +| 1 (current) | Low | Deploy, PostgreSQL backup | +| 2 (validation) | Low | None additional | +| 3 (resilience) | Low | Monitor DLQ depth | +| 4 (SQS) | Medium | SQS queue monitoring, IAM roles, DLQ alarms | +| 5 (security) | Medium | API key rotation, rate limit tuning | +| 6 (retention) | High | Partition management, S3 lifecycle, retention job monitoring, restore testing | +| 7 (observability) | Medium | Dashboard setup, alert thresholds, log aggregation | +| 8 (lichess) | Low | Additional API rate limit monitoring | +| 9 (re-analysis) | Medium | Batch job monitoring, schema migration coordination | + +### Recommended Implementation Order + +``` +Phase 2 (validation) ← Low effort, high safety impact +Phase 5 (security) ← Required before any public exposure +Phase 3 (resilience) ← Required before production indexing load +Phase 7 (observability) ← Required before diagnosing production issues +Phase 4 (SQS) ← Required before multi-instance deployment +Phase 8 (lichess) ← Feature expansion, independent of infra +Phase 6 (retention) ← Required once storage exceeds ~10GB +Phase 9 (re-analysis) ← Nice-to-have, depends on new motif demand +``` + +Each phase is independently deployable. No phase has a hard dependency on another, though the recommended order minimizes rework. diff --git a/jvm/src/main/java/com/muchq/indexer/engine/BUILD.bazel b/jvm/src/main/java/com/muchq/indexer/engine/BUILD.bazel new file mode 100644 index 00000000..59884e97 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/engine/BUILD.bazel @@ -0,0 +1,17 @@ +load("@rules_java//java:java_library.bzl", "java_library") + +java_library( + name = "engine", + srcs = [ + "FeatureExtractor.java", + "GameReplayer.java", + "PgnParser.java", + ], + visibility = ["//visibility:public"], + deps = [ + "//jvm/src/main/java/com/muchq/indexer/engine/model", + "//jvm/src/main/java/com/muchq/indexer/motifs", + "@maven//:io_github_tors42_chariot", + "@maven//:org_slf4j_slf4j_api", + ], +) diff --git a/jvm/src/main/java/com/muchq/indexer/engine/FeatureExtractor.java b/jvm/src/main/java/com/muchq/indexer/engine/FeatureExtractor.java new file mode 100644 index 00000000..552d376c --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/engine/FeatureExtractor.java @@ -0,0 +1,58 @@ +package com.muchq.indexer.engine; + +import com.muchq.indexer.engine.model.GameFeatures; +import com.muchq.indexer.engine.model.Motif; +import com.muchq.indexer.engine.model.ParsedGame; +import com.muchq.indexer.engine.model.PositionContext; +import com.muchq.indexer.motifs.MotifDetector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class FeatureExtractor { + private static final Logger LOG = LoggerFactory.getLogger(FeatureExtractor.class); + + private final PgnParser pgnParser; + private final GameReplayer replayer; + private final List detectors; + + public FeatureExtractor(PgnParser pgnParser, GameReplayer replayer, List detectors) { + this.pgnParser = pgnParser; + this.replayer = replayer; + this.detectors = detectors; + } + + public GameFeatures extract(String pgn) { + ParsedGame parsed = pgnParser.parse(pgn); + List positions; + try { + positions = replayer.replay(parsed.moveText()); + } catch (Exception e) { + LOG.warn("Failed to replay game, skipping motif detection", e); + return new GameFeatures(EnumSet.noneOf(Motif.class), 0, Map.of()); + } + + int numMoves = positions.isEmpty() ? 0 : positions.get(positions.size() - 1).moveNumber(); + Set foundMotifs = EnumSet.noneOf(Motif.class); + Map> allOccurrences = new EnumMap<>(Motif.class); + + for (MotifDetector detector : detectors) { + try { + List occurrences = detector.detect(positions); + if (!occurrences.isEmpty()) { + foundMotifs.add(detector.motif()); + allOccurrences.put(detector.motif(), occurrences); + } + } catch (Exception e) { + LOG.warn("Motif detector {} failed", detector.motif(), e); + } + } + + return new GameFeatures(foundMotifs, numMoves, allOccurrences); + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/engine/GameReplayer.java b/jvm/src/main/java/com/muchq/indexer/engine/GameReplayer.java new file mode 100644 index 00000000..52bf3102 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/engine/GameReplayer.java @@ -0,0 +1,55 @@ +package com.muchq.indexer.engine; + +import chariot.util.Board; +import com.muchq.indexer.engine.model.PositionContext; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class GameReplayer { + private static final Pattern MOVE_PATTERN = Pattern.compile( + "(?:\\d+\\.\\s*)?([KQRBNP]?[a-h]?[1-8]?x?[a-h][1-8](?:=[QRBN])?[+#]?|O-O-O|O-O)" + ); + + public List replay(String moveText) { + List positions = new ArrayList<>(); + Board board = Board.fromStandardPosition(); + + positions.add(new PositionContext(0, board.toFEN(), true)); + + List moves = extractMoves(moveText); + int moveNumber = 1; + boolean whiteToMove = true; + + for (String move : moves) { + board = board.play(move); + if (!whiteToMove) { + moveNumber++; + } + whiteToMove = !whiteToMove; + positions.add(new PositionContext(moveNumber, board.toFEN(), whiteToMove)); + } + + return positions; + } + + private List extractMoves(String moveText) { + List moves = new ArrayList<>(); + // Remove comments and variations + String cleaned = moveText.replaceAll("\\{[^}]*}", "") + .replaceAll("\\([^)]*\\)", "") + .replaceAll("\\$\\d+", ""); + + Matcher m = MOVE_PATTERN.matcher(cleaned); + while (m.find()) { + String move = m.group(1); + // Skip result indicators + if (!move.equals("1-0") && !move.equals("0-1") && !move.equals("1/2-1/2")) { + moves.add(move); + } + } + return moves; + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/engine/PgnParser.java b/jvm/src/main/java/com/muchq/indexer/engine/PgnParser.java new file mode 100644 index 00000000..3befb228 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/engine/PgnParser.java @@ -0,0 +1,45 @@ +package com.muchq.indexer.engine; + +import com.muchq.indexer.engine.model.ParsedGame; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class PgnParser { + private static final Pattern HEADER_PATTERN = Pattern.compile("\\[\\s*(\\w+)\\s+\"([^\"]*)\"\\s*]"); + + public ParsedGame parse(String pgn) { + Map headers = new LinkedHashMap<>(); + StringBuilder moveText = new StringBuilder(); + boolean inMoves = false; + + for (String line : pgn.split("\\r?\\n")) { + String trimmed = line.trim(); + if (trimmed.isEmpty()) { + if (!headers.isEmpty()) { + inMoves = true; + } + continue; + } + + if (!inMoves) { + Matcher m = HEADER_PATTERN.matcher(trimmed); + if (m.matches()) { + headers.put(m.group(1), m.group(2)); + continue; + } + } + + // If we've seen headers and this line doesn't match a header, it's movetext + inMoves = true; + if (!moveText.isEmpty()) { + moveText.append(' '); + } + moveText.append(trimmed); + } + + return new ParsedGame(headers, moveText.toString()); + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/engine/model/BUILD.bazel b/jvm/src/main/java/com/muchq/indexer/engine/model/BUILD.bazel new file mode 100644 index 00000000..e55c98a6 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/engine/model/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_java//java:java_library.bzl", "java_library") + +java_library( + name = "model", + srcs = [ + "GameFeatures.java", + "Motif.java", + "ParsedGame.java", + "PositionContext.java", + ], + visibility = ["//visibility:public"], +) diff --git a/jvm/src/main/java/com/muchq/indexer/engine/model/GameFeatures.java b/jvm/src/main/java/com/muchq/indexer/engine/model/GameFeatures.java new file mode 100644 index 00000000..2acf5077 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/engine/model/GameFeatures.java @@ -0,0 +1,17 @@ +package com.muchq.indexer.engine.model; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public record GameFeatures( + Set motifs, + int numMoves, + Map> occurrences +) { + public boolean hasMotif(Motif motif) { + return motifs.contains(motif); + } + + public record MotifOccurrence(int moveNumber, String description) {} +} diff --git a/jvm/src/main/java/com/muchq/indexer/engine/model/Motif.java b/jvm/src/main/java/com/muchq/indexer/engine/model/Motif.java new file mode 100644 index 00000000..c21a876f --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/engine/model/Motif.java @@ -0,0 +1,9 @@ +package com.muchq.indexer.engine.model; + +public enum Motif { + PIN, + CROSS_PIN, + FORK, + SKEWER, + DISCOVERED_ATTACK +} diff --git a/jvm/src/main/java/com/muchq/indexer/engine/model/ParsedGame.java b/jvm/src/main/java/com/muchq/indexer/engine/model/ParsedGame.java new file mode 100644 index 00000000..66b4d4e9 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/engine/model/ParsedGame.java @@ -0,0 +1,6 @@ +package com.muchq.indexer.engine.model; + +import java.util.Map; + +public record ParsedGame(Map headers, String moveText) { +} diff --git a/jvm/src/main/java/com/muchq/indexer/engine/model/PositionContext.java b/jvm/src/main/java/com/muchq/indexer/engine/model/PositionContext.java new file mode 100644 index 00000000..481a070e --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/engine/model/PositionContext.java @@ -0,0 +1,4 @@ +package com.muchq.indexer.engine.model; + +public record PositionContext(int moveNumber, String fen, boolean whiteToMove) { +} diff --git a/jvm/src/main/java/com/muchq/indexer/motifs/BUILD.bazel b/jvm/src/main/java/com/muchq/indexer/motifs/BUILD.bazel new file mode 100644 index 00000000..2362c116 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/motifs/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_java//java:java_library.bzl", "java_library") + +java_library( + name = "motifs", + srcs = [ + "CrossPinDetector.java", + "DiscoveredAttackDetector.java", + "ForkDetector.java", + "MotifDetector.java", + "PinDetector.java", + "SkewerDetector.java", + ], + visibility = ["//visibility:public"], + deps = [ + "//jvm/src/main/java/com/muchq/indexer/engine/model", + "@maven//:io_github_tors42_chariot", + ], +) diff --git a/jvm/src/main/java/com/muchq/indexer/motifs/CrossPinDetector.java b/jvm/src/main/java/com/muchq/indexer/motifs/CrossPinDetector.java new file mode 100644 index 00000000..6d4d6204 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/motifs/CrossPinDetector.java @@ -0,0 +1,106 @@ +package com.muchq.indexer.motifs; + +import com.muchq.indexer.engine.model.GameFeatures; +import com.muchq.indexer.engine.model.Motif; +import com.muchq.indexer.engine.model.PositionContext; + +import java.util.ArrayList; +import java.util.List; + +public class CrossPinDetector implements MotifDetector { + + @Override + public Motif motif() { + return Motif.CROSS_PIN; + } + + @Override + public List detect(List positions) { + List occurrences = new ArrayList<>(); + + for (PositionContext ctx : positions) { + String placement = ctx.fen().split(" ")[0]; + int[][] board = PinDetector.parsePlacement(placement); + + // A cross-pin occurs when a piece is pinned along two different directions + // simultaneously (e.g., pinned by a rook on a file AND a bishop on a diagonal). + if (hasCrossPin(board, ctx.whiteToMove())) { + occurrences.add(new GameFeatures.MotifOccurrence( + ctx.moveNumber(), "Cross-pin detected at move " + ctx.moveNumber())); + } + } + + return occurrences; + } + + private boolean hasCrossPin(int[][] board, boolean whiteToMove) { + int kingPiece = whiteToMove ? 6 : -6; + int kingRow = -1, kingCol = -1; + + for (int r = 0; r < 8; r++) { + for (int c = 0; c < 8; c++) { + if (board[r][c] == kingPiece) { + kingRow = r; + kingCol = c; + } + } + } + if (kingRow == -1) return false; + + // Find all pinned pieces and check if any piece is pinned along two axes + int[][] directions = {{0,1},{0,-1},{1,0},{-1,0},{1,1},{1,-1},{-1,1},{-1,-1}}; + List pinnedSquares = new ArrayList<>(); + + for (int[] dir : directions) { + int[] pinned = findPinnedPiece(board, kingRow, kingCol, dir[0], dir[1], whiteToMove); + if (pinned != null) { + pinnedSquares.add(pinned); + } + } + + // Check for duplicate pinned squares (same piece pinned from two directions) + for (int i = 0; i < pinnedSquares.size(); i++) { + for (int j = i + 1; j < pinnedSquares.size(); j++) { + if (pinnedSquares.get(i)[0] == pinnedSquares.get(j)[0] + && pinnedSquares.get(i)[1] == pinnedSquares.get(j)[1]) { + return true; + } + } + } + return false; + } + + private int[] findPinnedPiece(int[][] board, int kr, int kc, int dr, int dc, boolean whiteKing) { + int r = kr + dr, c = kc + dc; + int[] friendlyPos = null; + + while (r >= 0 && r < 8 && c >= 0 && c < 8) { + int piece = board[r][c]; + if (piece != 0) { + boolean isWhitePiece = piece > 0; + if (isWhitePiece == whiteKing) { + if (friendlyPos != null) return null; + friendlyPos = new int[]{r, c}; + } else { + if (friendlyPos != null && isSlidingAttacker(piece, dr, dc)) { + return friendlyPos; + } + return null; + } + } + r += dr; + c += dc; + } + return null; + } + + private boolean isSlidingAttacker(int piece, int dr, int dc) { + int absPiece = Math.abs(piece); + boolean isDiagonal = dr != 0 && dc != 0; + boolean isStraight = dr == 0 || dc == 0; + if (absPiece == 5) return true; + if (absPiece == 3 && isDiagonal) return true; + if (absPiece == 4 && isStraight) return true; + return false; + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/motifs/DiscoveredAttackDetector.java b/jvm/src/main/java/com/muchq/indexer/motifs/DiscoveredAttackDetector.java new file mode 100644 index 00000000..c7add08a --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/motifs/DiscoveredAttackDetector.java @@ -0,0 +1,111 @@ +package com.muchq.indexer.motifs; + +import com.muchq.indexer.engine.model.GameFeatures; +import com.muchq.indexer.engine.model.Motif; +import com.muchq.indexer.engine.model.PositionContext; + +import java.util.ArrayList; +import java.util.List; + +public class DiscoveredAttackDetector implements MotifDetector { + + @Override + public Motif motif() { + return Motif.DISCOVERED_ATTACK; + } + + @Override + public List detect(List positions) { + List occurrences = new ArrayList<>(); + + // Compare consecutive positions to detect discovered attacks. + // A discovered attack occurs when a piece moves and reveals an attack + // from a sliding piece behind it. + for (int i = 1; i < positions.size(); i++) { + PositionContext before = positions.get(i - 1); + PositionContext after = positions.get(i); + + String beforePlacement = before.fen().split(" ")[0]; + String afterPlacement = after.fen().split(" ")[0]; + int[][] boardBefore = PinDetector.parsePlacement(beforePlacement); + int[][] boardAfter = PinDetector.parsePlacement(afterPlacement); + + // The side that just moved is the opposite of whose turn it now is + boolean moverIsWhite = after.whiteToMove() ? false : true; + + if (hasDiscoveredAttack(boardBefore, boardAfter, moverIsWhite)) { + occurrences.add(new GameFeatures.MotifOccurrence( + after.moveNumber(), "Discovered attack at move " + after.moveNumber())); + } + } + + return occurrences; + } + + private boolean hasDiscoveredAttack(int[][] before, int[][] after, boolean moverIsWhite) { + // Find the piece that moved (square that became empty) + for (int r = 0; r < 8; r++) { + for (int c = 0; c < 8; c++) { + int pieceBefore = before[r][c]; + int pieceAfter = after[r][c]; + + if (pieceBefore != 0 && pieceAfter == 0) { + boolean isWhite = pieceBefore > 0; + if (isWhite == moverIsWhite) { + // This square was vacated by the moving piece. + // Check if any sliding piece behind it now has a new attack line. + if (revealsAttack(after, r, c, moverIsWhite)) { + return true; + } + } + } + } + } + return false; + } + + private boolean revealsAttack(int[][] board, int vacatedR, int vacatedC, boolean moverIsWhite) { + int[][] directions = {{0,1},{0,-1},{1,0},{-1,0},{1,1},{1,-1},{-1,1},{-1,-1}}; + + for (int[] dir : directions) { + // Look behind the vacated square for a friendly sliding piece + int br = vacatedR - dir[0], bc = vacatedC - dir[1]; + while (br >= 0 && br < 8 && bc >= 0 && bc < 8) { + int piece = board[br][bc]; + if (piece != 0) { + boolean isWhite = piece > 0; + if (isWhite == moverIsWhite && isSlidingAttacker(piece, dir)) { + // Check if there's an enemy piece along the forward direction + int fr = vacatedR + dir[0], fc = vacatedC + dir[1]; + while (fr >= 0 && fr < 8 && fc >= 0 && fc < 8) { + int target = board[fr][fc]; + if (target != 0) { + boolean targetIsWhite = target > 0; + if (targetIsWhite != moverIsWhite && Math.abs(target) >= 2) { + return true; + } + break; + } + fr += dir[0]; + fc += dir[1]; + } + } + break; + } + br -= dir[0]; + bc -= dir[1]; + } + } + return false; + } + + private boolean isSlidingAttacker(int piece, int[] dir) { + int absPiece = Math.abs(piece); + boolean isDiagonal = dir[0] != 0 && dir[1] != 0; + boolean isStraight = dir[0] == 0 || dir[1] == 0; + if (absPiece == 5) return true; + if (absPiece == 3 && isDiagonal) return true; + if (absPiece == 4 && isStraight) return true; + return false; + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/motifs/ForkDetector.java b/jvm/src/main/java/com/muchq/indexer/motifs/ForkDetector.java new file mode 100644 index 00000000..4f936737 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/motifs/ForkDetector.java @@ -0,0 +1,118 @@ +package com.muchq.indexer.motifs; + +import com.muchq.indexer.engine.model.GameFeatures; +import com.muchq.indexer.engine.model.Motif; +import com.muchq.indexer.engine.model.PositionContext; + +import java.util.ArrayList; +import java.util.List; + +public class ForkDetector implements MotifDetector { + + @Override + public Motif motif() { + return Motif.FORK; + } + + @Override + public List detect(List positions) { + List occurrences = new ArrayList<>(); + + for (PositionContext ctx : positions) { + String placement = ctx.fen().split(" ")[0]; + int[][] board = PinDetector.parsePlacement(placement); + + // Check if any piece attacks two or more enemy pieces of significant value + if (hasFork(board, !ctx.whiteToMove())) { + occurrences.add(new GameFeatures.MotifOccurrence( + ctx.moveNumber(), "Fork detected at move " + ctx.moveNumber())); + } + } + + return occurrences; + } + + private boolean hasFork(int[][] board, boolean attackerIsWhite) { + for (int r = 0; r < 8; r++) { + for (int c = 0; c < 8; c++) { + int piece = board[r][c]; + if (piece == 0) continue; + boolean isWhite = piece > 0; + if (isWhite != attackerIsWhite) continue; + + int absPiece = Math.abs(piece); + List attacked = getAttackedSquares(board, r, c, absPiece, attackerIsWhite); + + // Count how many valuable enemy pieces are attacked + int valuableTargets = 0; + for (int[] sq : attacked) { + int target = board[sq[0]][sq[1]]; + if (target != 0 && (target > 0) != attackerIsWhite) { + int targetValue = Math.abs(target); + // Target must be at least a knight/bishop (value >= 2) + if (targetValue >= 2) { + valuableTargets++; + } + } + } + if (valuableTargets >= 2) return true; + } + } + return false; + } + + private List getAttackedSquares(int[][] board, int r, int c, int pieceType, boolean isWhite) { + List squares = new ArrayList<>(); + switch (pieceType) { + case 2 -> addKnightAttacks(r, c, squares); // Knight + case 1 -> addPawnAttacks(r, c, isWhite, squares); // Pawn + case 3 -> addSlidingAttacks(board, r, c, new int[][]{{1,1},{1,-1},{-1,1},{-1,-1}}, squares); // Bishop + case 4 -> addSlidingAttacks(board, r, c, new int[][]{{0,1},{0,-1},{1,0},{-1,0}}, squares); // Rook + case 5 -> { // Queen + addSlidingAttacks(board, r, c, new int[][]{{0,1},{0,-1},{1,0},{-1,0},{1,1},{1,-1},{-1,1},{-1,-1}}, squares); + } + case 6 -> addKingAttacks(r, c, squares); // King + } + return squares; + } + + private void addKnightAttacks(int r, int c, List squares) { + int[][] offsets = {{-2,-1},{-2,1},{-1,-2},{-1,2},{1,-2},{1,2},{2,-1},{2,1}}; + for (int[] off : offsets) { + int nr = r + off[0], nc = c + off[1]; + if (nr >= 0 && nr < 8 && nc >= 0 && nc < 8) { + squares.add(new int[]{nr, nc}); + } + } + } + + private void addPawnAttacks(int r, int c, boolean isWhite, List squares) { + int dir = isWhite ? -1 : 1; + if (c > 0 && r + dir >= 0 && r + dir < 8) squares.add(new int[]{r + dir, c - 1}); + if (c < 7 && r + dir >= 0 && r + dir < 8) squares.add(new int[]{r + dir, c + 1}); + } + + private void addKingAttacks(int r, int c, List squares) { + for (int dr = -1; dr <= 1; dr++) { + for (int dc = -1; dc <= 1; dc++) { + if (dr == 0 && dc == 0) continue; + int nr = r + dr, nc = c + dc; + if (nr >= 0 && nr < 8 && nc >= 0 && nc < 8) { + squares.add(new int[]{nr, nc}); + } + } + } + } + + private void addSlidingAttacks(int[][] board, int r, int c, int[][] directions, List squares) { + for (int[] dir : directions) { + int nr = r + dir[0], nc = c + dir[1]; + while (nr >= 0 && nr < 8 && nc >= 0 && nc < 8) { + squares.add(new int[]{nr, nc}); + if (board[nr][nc] != 0) break; // blocked + nr += dir[0]; + nc += dir[1]; + } + } + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/motifs/MotifDetector.java b/jvm/src/main/java/com/muchq/indexer/motifs/MotifDetector.java new file mode 100644 index 00000000..d0a1f811 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/motifs/MotifDetector.java @@ -0,0 +1,12 @@ +package com.muchq.indexer.motifs; + +import com.muchq.indexer.engine.model.GameFeatures; +import com.muchq.indexer.engine.model.Motif; +import com.muchq.indexer.engine.model.PositionContext; + +import java.util.List; + +public interface MotifDetector { + Motif motif(); + List detect(List positions); +} diff --git a/jvm/src/main/java/com/muchq/indexer/motifs/PinDetector.java b/jvm/src/main/java/com/muchq/indexer/motifs/PinDetector.java new file mode 100644 index 00000000..9f82db1c --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/motifs/PinDetector.java @@ -0,0 +1,159 @@ +package com.muchq.indexer.motifs; + +import chariot.util.Board; +import com.muchq.indexer.engine.model.GameFeatures; +import com.muchq.indexer.engine.model.Motif; +import com.muchq.indexer.engine.model.PositionContext; + +import java.util.ArrayList; +import java.util.List; + +public class PinDetector implements MotifDetector { + + @Override + public Motif motif() { + return Motif.PIN; + } + + @Override + public List detect(List positions) { + List occurrences = new ArrayList<>(); + + for (PositionContext ctx : positions) { + Board board = Board.fromFEN(ctx.fen()); + // A pin exists when a sliding piece (bishop, rook, queen) attacks a piece + // that shields a more valuable piece behind it along the same line. + // We detect this by looking for pieces that have limited legal moves + // compared to their pseudo-legal moves due to being pinned to the king. + if (hasPinnedPiece(board, ctx.whiteToMove())) { + occurrences.add(new GameFeatures.MotifOccurrence( + ctx.moveNumber(), "Pin detected at move " + ctx.moveNumber())); + } + } + + return occurrences; + } + + private boolean hasPinnedPiece(Board board, boolean whiteToMove) { + // Use chariot's legal move generation to detect pins: + // A piece is pinned if the number of legal moves is restricted. + // We compare legal moves count for each piece vs expected mobility. + // For simplicity, we detect pin by checking if any piece of the side to move + // has zero legal moves while not being blocked by friendly pieces alone. + String fen = board.toFEN(); + String[] parts = fen.split(" "); + String placement = parts[0]; + + // Simple heuristic: check if there's a piece on a line between an attacker + // and the king. This is a simplified detection. + return detectPinFromFen(placement, whiteToMove); + } + + private boolean detectPinFromFen(String placement, boolean whiteToMove) { + // Find king position and check for pieces on diagonals/files/ranks + // between sliding attackers and the king. + // This is a heuristic approach - full implementation would use + // ray-casting from the king position. + int[][] boardArray = parsePlacement(placement); + int kingRow = -1, kingCol = -1; + int kingPiece = whiteToMove ? 6 : -6; // K=6, k=-6 + + for (int r = 0; r < 8; r++) { + for (int c = 0; c < 8; c++) { + if (boardArray[r][c] == kingPiece) { + kingRow = r; + kingCol = c; + } + } + } + + if (kingRow == -1) return false; + + // Check all 8 directions from the king for pins + int[][] directions = {{0,1},{0,-1},{1,0},{-1,0},{1,1},{1,-1},{-1,1},{-1,-1}}; + for (int[] dir : directions) { + if (isPinAlongRay(boardArray, kingRow, kingCol, dir[0], dir[1], whiteToMove)) { + return true; + } + } + return false; + } + + private boolean isPinAlongRay(int[][] board, int kr, int kc, int dr, int dc, boolean whiteKing) { + int friendlyPieceCount = 0; + int r = kr + dr, c = kc + dc; + boolean foundFriendly = false; + + while (r >= 0 && r < 8 && c >= 0 && c < 8) { + int piece = board[r][c]; + if (piece != 0) { + boolean isWhitePiece = piece > 0; + if (isWhitePiece == whiteKing) { + // Friendly piece + friendlyPieceCount++; + if (friendlyPieceCount > 1) return false; + foundFriendly = true; + } else { + // Enemy piece - check if it's a sliding attacker on this line + if (foundFriendly && isSlidingAttacker(piece, dr, dc)) { + return true; + } + return false; + } + } + r += dr; + c += dc; + } + return false; + } + + private boolean isSlidingAttacker(int piece, int dr, int dc) { + int absPiece = Math.abs(piece); + boolean isDiagonal = dr != 0 && dc != 0; + boolean isStraight = dr == 0 || dc == 0; + + // Queen (5) attacks on both diagonals and straights + if (absPiece == 5) return true; + // Bishop (3) attacks on diagonals + if (absPiece == 3 && isDiagonal) return true; + // Rook (4) attacks on straight lines + if (absPiece == 4 && isStraight) return true; + + return false; + } + + static int[][] parsePlacement(String placement) { + int[][] board = new int[8][8]; + String[] ranks = placement.split("/"); + for (int r = 0; r < 8; r++) { + int c = 0; + for (char ch : ranks[r].toCharArray()) { + if (Character.isDigit(ch)) { + c += ch - '0'; + } else { + board[r][c] = pieceValue(ch); + c++; + } + } + } + return board; + } + + static int pieceValue(char ch) { + return switch (ch) { + case 'K' -> 6; + case 'Q' -> 5; + case 'R' -> 4; + case 'B' -> 3; + case 'N' -> 2; + case 'P' -> 1; + case 'k' -> -6; + case 'q' -> -5; + case 'r' -> -4; + case 'b' -> -3; + case 'n' -> -2; + case 'p' -> -1; + default -> 0; + }; + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/motifs/SkewerDetector.java b/jvm/src/main/java/com/muchq/indexer/motifs/SkewerDetector.java new file mode 100644 index 00000000..ae8c3038 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/motifs/SkewerDetector.java @@ -0,0 +1,93 @@ +package com.muchq.indexer.motifs; + +import com.muchq.indexer.engine.model.GameFeatures; +import com.muchq.indexer.engine.model.Motif; +import com.muchq.indexer.engine.model.PositionContext; + +import java.util.ArrayList; +import java.util.List; + +public class SkewerDetector implements MotifDetector { + + @Override + public Motif motif() { + return Motif.SKEWER; + } + + @Override + public List detect(List positions) { + List occurrences = new ArrayList<>(); + + for (PositionContext ctx : positions) { + String placement = ctx.fen().split(" ")[0]; + int[][] board = PinDetector.parsePlacement(placement); + + // A skewer is the opposite of a pin: a more valuable piece is in front, + // and when it moves, a less valuable piece behind is captured. + if (hasSkewer(board, !ctx.whiteToMove())) { + occurrences.add(new GameFeatures.MotifOccurrence( + ctx.moveNumber(), "Skewer detected at move " + ctx.moveNumber())); + } + } + + return occurrences; + } + + private boolean hasSkewer(int[][] board, boolean attackerIsWhite) { + int[][] directions = {{0,1},{0,-1},{1,0},{-1,0},{1,1},{1,-1},{-1,1},{-1,-1}}; + + for (int r = 0; r < 8; r++) { + for (int c = 0; c < 8; c++) { + int piece = board[r][c]; + if (piece == 0) continue; + boolean isWhite = piece > 0; + if (isWhite != attackerIsWhite) continue; + + int absPiece = Math.abs(piece); + // Only sliding pieces can skewer + if (absPiece != 3 && absPiece != 4 && absPiece != 5) continue; + + for (int[] dir : directions) { + if (!canAttackDirection(absPiece, dir)) continue; + if (isSkewerAlongRay(board, r, c, dir[0], dir[1], attackerIsWhite)) { + return true; + } + } + } + } + return false; + } + + private boolean canAttackDirection(int absPiece, int[] dir) { + boolean isDiagonal = dir[0] != 0 && dir[1] != 0; + boolean isStraight = dir[0] == 0 || dir[1] == 0; + if (absPiece == 5) return true; // Queen + if (absPiece == 3) return isDiagonal; // Bishop + if (absPiece == 4) return isStraight; // Rook + return false; + } + + private boolean isSkewerAlongRay(int[][] board, int ar, int ac, int dr, int dc, boolean attackerIsWhite) { + int r = ar + dr, c = ac + dc; + int firstValue = -1; + + while (r >= 0 && r < 8 && c >= 0 && c < 8) { + int piece = board[r][c]; + if (piece != 0) { + boolean isWhite = piece > 0; + if (isWhite == attackerIsWhite) return false; // friendly piece blocks + + int value = Math.abs(piece); + if (firstValue == -1) { + firstValue = value; + } else { + // Skewer: first piece (in front) is more valuable than second + return firstValue > value && value >= 2; + } + } + r += dr; + c += dc; + } + return false; + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/queue/BUILD.bazel b/jvm/src/main/java/com/muchq/indexer/queue/BUILD.bazel new file mode 100644 index 00000000..5d8b0365 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/queue/BUILD.bazel @@ -0,0 +1,11 @@ +load("@rules_java//java:java_library.bzl", "java_library") + +java_library( + name = "queue", + srcs = [ + "InMemoryIndexQueue.java", + "IndexMessage.java", + "IndexQueue.java", + ], + visibility = ["//visibility:public"], +) diff --git a/jvm/src/main/java/com/muchq/indexer/queue/InMemoryIndexQueue.java b/jvm/src/main/java/com/muchq/indexer/queue/InMemoryIndexQueue.java new file mode 100644 index 00000000..6b95cef4 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/queue/InMemoryIndexQueue.java @@ -0,0 +1,31 @@ +package com.muchq.indexer.queue; + +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +public class InMemoryIndexQueue implements IndexQueue { + private final LinkedBlockingQueue queue = new LinkedBlockingQueue<>(); + + @Override + public void enqueue(IndexMessage message) { + queue.add(message); + } + + @Override + public Optional poll(Duration timeout) { + try { + IndexMessage msg = queue.poll(timeout.toMillis(), TimeUnit.MILLISECONDS); + return Optional.ofNullable(msg); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return Optional.empty(); + } + } + + @Override + public int size() { + return queue.size(); + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/queue/IndexMessage.java b/jvm/src/main/java/com/muchq/indexer/queue/IndexMessage.java new file mode 100644 index 00000000..5a391e6c --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/queue/IndexMessage.java @@ -0,0 +1,6 @@ +package com.muchq.indexer.queue; + +import java.util.UUID; + +public record IndexMessage(UUID requestId, String player, String platform, String startMonth, String endMonth) { +} diff --git a/jvm/src/main/java/com/muchq/indexer/queue/IndexQueue.java b/jvm/src/main/java/com/muchq/indexer/queue/IndexQueue.java new file mode 100644 index 00000000..028570e7 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/queue/IndexQueue.java @@ -0,0 +1,10 @@ +package com.muchq.indexer.queue; + +import java.time.Duration; +import java.util.Optional; + +public interface IndexQueue { + void enqueue(IndexMessage message); + Optional poll(Duration timeout); + int size(); +} diff --git a/jvm/src/main/java/com/muchq/indexer/worker/BUILD.bazel b/jvm/src/main/java/com/muchq/indexer/worker/BUILD.bazel new file mode 100644 index 00000000..8bbe4612 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/worker/BUILD.bazel @@ -0,0 +1,25 @@ +load("@rules_java//java:java_library.bzl", "java_library") + +java_library( + name = "worker", + srcs = [ + "IndexWorker.java", + "IndexWorkerLifecycle.java", + "ResultMapper.java", + ], + visibility = ["//visibility:public"], + deps = [ + "//jvm/src/main/java/com/muchq/chess_com_api", + "//jvm/src/main/java/com/muchq/indexer/db", + "//jvm/src/main/java/com/muchq/indexer/engine", + "//jvm/src/main/java/com/muchq/indexer/engine/model", + "//jvm/src/main/java/com/muchq/indexer/queue", + "@maven//:com_fasterxml_jackson_core_jackson_core", + "@maven//:com_fasterxml_jackson_core_jackson_databind", + "@maven//:io_micronaut_micronaut_context", + "@maven//:io_micronaut_micronaut_http_server_netty", + "@maven//:io_micronaut_micronaut_inject", + "@maven//:io_micronaut_micronaut_runtime", + "@maven//:org_slf4j_slf4j_api", + ], +) diff --git a/jvm/src/main/java/com/muchq/indexer/worker/IndexWorker.java b/jvm/src/main/java/com/muchq/indexer/worker/IndexWorker.java new file mode 100644 index 00000000..98e712f2 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/worker/IndexWorker.java @@ -0,0 +1,135 @@ +package com.muchq.indexer.worker; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.muchq.chess_com_api.ChessClient; +import com.muchq.chess_com_api.GamesResponse; +import com.muchq.chess_com_api.PlayedGame; +import com.muchq.indexer.db.GameFeatureStore; +import com.muchq.indexer.db.IndexingRequestStore; +import com.muchq.indexer.engine.FeatureExtractor; +import com.muchq.indexer.engine.model.GameFeatures; +import com.muchq.indexer.engine.model.Motif; +import com.muchq.indexer.queue.IndexMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.time.YearMonth; +import java.time.format.DateTimeFormatter; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class IndexWorker { + private static final Logger LOG = LoggerFactory.getLogger(IndexWorker.class); + private static final DateTimeFormatter MONTH_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM"); + private static final Pattern ECO_PATTERN = Pattern.compile("\\[ECO\\s+\"([^\"]+)\"\\]"); + + private final ChessClient chessClient; + private final FeatureExtractor featureExtractor; + private final IndexingRequestStore requestStore; + private final GameFeatureStore gameFeatureStore; + private final ObjectMapper objectMapper; + + public IndexWorker( + ChessClient chessClient, + FeatureExtractor featureExtractor, + IndexingRequestStore requestStore, + GameFeatureStore gameFeatureStore, + ObjectMapper objectMapper) { + this.chessClient = chessClient; + this.featureExtractor = featureExtractor; + this.requestStore = requestStore; + this.gameFeatureStore = gameFeatureStore; + this.objectMapper = objectMapper; + } + + public void process(IndexMessage message) { + LOG.info("Processing index request {} for player={} platform={}", + message.requestId(), message.player(), message.platform()); + + try { + requestStore.updateStatus(message.requestId(), "PROCESSING", null, 0); + + YearMonth start = YearMonth.parse(message.startMonth(), MONTH_FORMAT); + YearMonth end = YearMonth.parse(message.endMonth(), MONTH_FORMAT); + int totalIndexed = 0; + + for (YearMonth month = start; !month.isAfter(end); month = month.plusMonths(1)) { + Optional response = chessClient.fetchGames(message.player(), month); + if (response.isEmpty()) { + LOG.warn("No games found for player={} month={}", message.player(), month); + continue; + } + + for (PlayedGame game : response.get().games()) { + try { + indexGame(message, game); + totalIndexed++; + } catch (Exception e) { + LOG.warn("Failed to index game {}", game.url(), e); + } + } + + requestStore.updateStatus(message.requestId(), "PROCESSING", null, totalIndexed); + } + + requestStore.updateStatus(message.requestId(), "COMPLETED", null, totalIndexed); + LOG.info("Completed indexing request {} with {} games", message.requestId(), totalIndexed); + } catch (Exception e) { + LOG.error("Failed to process index request {}", message.requestId(), e); + requestStore.updateStatus(message.requestId(), "FAILED", e.getMessage(), 0); + } + } + + private void indexGame(IndexMessage message, PlayedGame game) { + GameFeatures features = featureExtractor.extract(game.pgn()); + + String motifsJson; + try { + motifsJson = objectMapper.writeValueAsString(features.occurrences()); + } catch (JsonProcessingException e) { + motifsJson = "{}"; + } + + String result = determineResult(game); + + GameFeatureStore.GameFeature row = new GameFeatureStore.GameFeature( + null, // id generated by DB + message.requestId(), + game.url(), + message.platform(), + game.whiteResult() != null ? game.whiteResult().username() : null, + game.blackResult() != null ? game.blackResult().username() : null, + game.whiteResult() != null ? Integer.valueOf(game.whiteResult().rating()) : null, + game.blackResult() != null ? Integer.valueOf(game.blackResult().rating()) : null, + game.timeClass(), + extractEcoFromPgn(game.pgn()), + result, + game.endTime(), + features.numMoves(), + features.hasMotif(Motif.PIN), + features.hasMotif(Motif.CROSS_PIN), + features.hasMotif(Motif.FORK), + features.hasMotif(Motif.SKEWER), + features.hasMotif(Motif.DISCOVERED_ATTACK), + motifsJson, + game.pgn() + ); + + gameFeatureStore.insert(row); + } + + private String determineResult(PlayedGame game) { + String whiteResult = game.whiteResult() != null ? game.whiteResult().result() : null; + String blackResult = game.blackResult() != null ? game.blackResult().result() : null; + return ResultMapper.mapResult(whiteResult, blackResult); + } + + private String extractEcoFromPgn(String pgn) { + Matcher m = ECO_PATTERN.matcher(pgn); + return m.find() ? m.group(1) : null; + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/worker/IndexWorkerLifecycle.java b/jvm/src/main/java/com/muchq/indexer/worker/IndexWorkerLifecycle.java new file mode 100644 index 00000000..de6a57a6 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/worker/IndexWorkerLifecycle.java @@ -0,0 +1,47 @@ +package com.muchq.indexer.worker; + +import com.muchq.indexer.queue.IndexMessage; +import com.muchq.indexer.queue.IndexQueue; +import io.micronaut.context.event.ApplicationEventListener; +import io.micronaut.runtime.server.event.ServerStartupEvent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.Optional; + +public class IndexWorkerLifecycle implements ApplicationEventListener { + private static final Logger LOG = LoggerFactory.getLogger(IndexWorkerLifecycle.class); + + private final IndexQueue queue; + private final IndexWorker worker; + private volatile boolean running = true; + + public IndexWorkerLifecycle(IndexQueue queue, IndexWorker worker) { + this.queue = queue; + this.worker = worker; + } + + @Override + public void onApplicationEvent(ServerStartupEvent event) { + Thread workerThread = new Thread(this::pollLoop, "index-worker"); + workerThread.setDaemon(true); + workerThread.start(); + LOG.info("Index worker started"); + } + + private void pollLoop() { + while (running) { + try { + Optional message = queue.poll(Duration.ofSeconds(5)); + message.ifPresent(worker::process); + } catch (Exception e) { + LOG.error("Error in index worker poll loop", e); + } + } + } + + public void stop() { + running = false; + } +} diff --git a/jvm/src/main/java/com/muchq/indexer/worker/ResultMapper.java b/jvm/src/main/java/com/muchq/indexer/worker/ResultMapper.java new file mode 100644 index 00000000..c9e981b5 --- /dev/null +++ b/jvm/src/main/java/com/muchq/indexer/worker/ResultMapper.java @@ -0,0 +1,74 @@ +package com.muchq.indexer.worker; + +/** + * Maps chess.com API result values to standard chess notation. + * + * Chess.com returns separate result strings for white and black: + * - "win" for the winning side + * - "resigned", "checkmated", "timeout", "abandoned" for the losing side + * - "agreed", "repetition", "stalemate", "insufficient", "50move", "timevsinsufficient" for draws + * + * This class normalizes these to standard notation: "1-0", "0-1", "1/2-1/2", or "unknown". + */ +public class ResultMapper { + + /** + * Determines the game result in standard notation. + * + * @param whiteResult chess.com result string for white (may be null) + * @param blackResult chess.com result string for black (may be null) + * @return "1-0" (white wins), "0-1" (black wins), "1/2-1/2" (draw), or "unknown" + */ + public static String mapResult(String whiteResult, String blackResult) { + if (whiteResult == null && blackResult == null) { + return "unknown"; + } + + // Explicit win + if ("win".equals(whiteResult)) { + return "1-0"; + } + if ("win".equals(blackResult)) { + return "0-1"; + } + + // Draw results + if (isDrawResult(whiteResult) || isDrawResult(blackResult)) { + return "1/2-1/2"; + } + + // White lost → black won + if (isLossResult(whiteResult)) { + return "0-1"; + } + // Black lost → white won + if (isLossResult(blackResult)) { + return "1-0"; + } + + return "unknown"; + } + + /** + * Checks if the result string indicates a draw. + */ + public static boolean isDrawResult(String result) { + if (result == null) return false; + return switch (result) { + case "agreed", "repetition", "stalemate", "insufficient", + "50move", "timevsinsufficient", "drawn" -> true; + default -> false; + }; + } + + /** + * Checks if the result string indicates a loss for that player. + */ + public static boolean isLossResult(String result) { + if (result == null) return false; + return switch (result) { + case "resigned", "checkmated", "timeout", "abandoned", "lose" -> true; + default -> false; + }; + } +} diff --git a/jvm/src/test/java/com/muchq/indexer/chessql/compiler/BUILD.bazel b/jvm/src/test/java/com/muchq/indexer/chessql/compiler/BUILD.bazel new file mode 100644 index 00000000..71a51dac --- /dev/null +++ b/jvm/src/test/java/com/muchq/indexer/chessql/compiler/BUILD.bazel @@ -0,0 +1,14 @@ +load("@contrib_rules_jvm//java:defs.bzl", "java_test_suite") + +java_test_suite( + name = "compiler", + size = "small", + srcs = ["SqlCompilerTest.java"], + deps = [ + "//jvm/src/main/java/com/muchq/indexer/chessql/ast", + "//jvm/src/main/java/com/muchq/indexer/chessql/compiler", + "//jvm/src/main/java/com/muchq/indexer/chessql/parser", + "@maven//:junit_junit", + "@maven//:org_assertj_assertj_core", + ], +) diff --git a/jvm/src/test/java/com/muchq/indexer/chessql/compiler/SqlCompilerTest.java b/jvm/src/test/java/com/muchq/indexer/chessql/compiler/SqlCompilerTest.java new file mode 100644 index 00000000..6e8861cf --- /dev/null +++ b/jvm/src/test/java/com/muchq/indexer/chessql/compiler/SqlCompilerTest.java @@ -0,0 +1,104 @@ +package com.muchq.indexer.chessql.compiler; + +import com.muchq.indexer.chessql.ast.Expr; +import com.muchq.indexer.chessql.parser.Parser; +import org.junit.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class SqlCompilerTest { + + private final SqlCompiler compiler = new SqlCompiler(); + + @Test + public void testSimpleComparison() { + CompiledQuery result = compile("white.elo >= 2500"); + assertThat(result.sql()).isEqualTo("white_elo >= ?"); + assertThat(result.parameters()).isEqualTo(List.of(2500)); + } + + @Test + public void testMotif() { + CompiledQuery result = compile("motif(fork)"); + assertThat(result.sql()).isEqualTo("has_fork = TRUE"); + assertThat(result.parameters()).isEmpty(); + } + + @Test + public void testAndExpression() { + CompiledQuery result = compile("white.elo >= 2500 AND motif(fork)"); + assertThat(result.sql()).isEqualTo("(white_elo >= ? AND has_fork = TRUE)"); + assertThat(result.parameters()).isEqualTo(List.of(2500)); + } + + @Test + public void testOrExpression() { + CompiledQuery result = compile("motif(fork) OR motif(pin)"); + assertThat(result.sql()).isEqualTo("(has_fork = TRUE OR has_pin = TRUE)"); + assertThat(result.parameters()).isEmpty(); + } + + @Test + public void testNotExpression() { + CompiledQuery result = compile("NOT motif(pin)"); + assertThat(result.sql()).isEqualTo("(NOT has_pin = TRUE)"); + assertThat(result.parameters()).isEmpty(); + } + + @Test + public void testInExpression() { + CompiledQuery result = compile("platform IN [\"lichess\", \"chess.com\"]"); + assertThat(result.sql()).isEqualTo("platform IN (?, ?)"); + assertThat(result.parameters()).isEqualTo(List.of("lichess", "chess.com")); + } + + @Test + public void testComplexQuery() { + CompiledQuery result = compile("white.elo >= 2500 AND motif(cross_pin)"); + assertThat(result.sql()).isEqualTo("(white_elo >= ? AND has_cross_pin = TRUE)"); + assertThat(result.parameters()).isEqualTo(List.of(2500)); + } + + @Test + public void testEndToEnd() { + CompiledQuery result = compile("white.elo >= 2500 AND motif(fork)"); + assertThat(result.sql()).isEqualTo("(white_elo >= ? AND has_fork = TRUE)"); + assertThat(result.parameters()).isEqualTo(List.of(2500)); + } + + @Test + public void testNestedBooleans() { + CompiledQuery result = compile("(motif(fork) OR motif(pin)) AND white.elo > 2000"); + assertThat(result.sql()).isEqualTo("((has_fork = TRUE OR has_pin = TRUE) AND white_elo > ?)"); + assertThat(result.parameters()).isEqualTo(List.of(2000)); + } + + @Test + public void testUnknownMotif() { + assertThatThrownBy(() -> compile("motif(unknown)")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Unknown motif"); + } + + @Test + public void testUnknownField() { + assertThatThrownBy(() -> compile("bogus_field >= 100")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Unknown field"); + } + + @Test + public void testDirectColumnName() { + CompiledQuery result = compile("white_elo >= 2500"); + assertThat(result.sql()).isEqualTo("white_elo >= ?"); + assertThat(result.parameters()).isEqualTo(List.of(2500)); + } + + private CompiledQuery compile(String input) { + Expr expr = Parser.parse(input); + return compiler.compile(expr); + } +} diff --git a/jvm/src/test/java/com/muchq/indexer/chessql/lexer/BUILD.bazel b/jvm/src/test/java/com/muchq/indexer/chessql/lexer/BUILD.bazel new file mode 100644 index 00000000..1f772d98 --- /dev/null +++ b/jvm/src/test/java/com/muchq/indexer/chessql/lexer/BUILD.bazel @@ -0,0 +1,12 @@ +load("@contrib_rules_jvm//java:defs.bzl", "java_test_suite") + +java_test_suite( + name = "lexer", + size = "small", + srcs = ["LexerTest.java"], + deps = [ + "//jvm/src/main/java/com/muchq/indexer/chessql/lexer", + "@maven//:junit_junit", + "@maven//:org_assertj_assertj_core", + ], +) diff --git a/jvm/src/test/java/com/muchq/indexer/chessql/lexer/LexerTest.java b/jvm/src/test/java/com/muchq/indexer/chessql/lexer/LexerTest.java new file mode 100644 index 00000000..f9e590ef --- /dev/null +++ b/jvm/src/test/java/com/muchq/indexer/chessql/lexer/LexerTest.java @@ -0,0 +1,91 @@ +package com.muchq.indexer.chessql.lexer; + +import org.junit.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class LexerTest { + + @Test + public void testSimpleComparison() { + List tokens = new Lexer("white_elo >= 2500").tokenize(); + assertThat(tokens).hasSize(4); // IDENTIFIER, GTE, NUMBER, EOF + assertThat(tokens.get(0).type()).isEqualTo(TokenType.IDENTIFIER); + assertThat(tokens.get(0).value()).isEqualTo("white_elo"); + assertThat(tokens.get(1).type()).isEqualTo(TokenType.GTE); + assertThat(tokens.get(2).type()).isEqualTo(TokenType.NUMBER); + assertThat(tokens.get(2).value()).isEqualTo("2500"); + assertThat(tokens.get(3).type()).isEqualTo(TokenType.EOF); + } + + @Test + public void testMotifExpression() { + List tokens = new Lexer("motif(fork)").tokenize(); + assertThat(tokens).hasSize(5); // MOTIF, LPAREN, IDENTIFIER, RPAREN, EOF + assertThat(tokens.get(0).type()).isEqualTo(TokenType.MOTIF); + assertThat(tokens.get(1).type()).isEqualTo(TokenType.LPAREN); + assertThat(tokens.get(2).type()).isEqualTo(TokenType.IDENTIFIER); + assertThat(tokens.get(2).value()).isEqualTo("fork"); + assertThat(tokens.get(3).type()).isEqualTo(TokenType.RPAREN); + } + + @Test + public void testKeywords() { + List tokens = new Lexer("AND OR NOT IN motif").tokenize(); + assertThat(tokens.get(0).type()).isEqualTo(TokenType.AND); + assertThat(tokens.get(1).type()).isEqualTo(TokenType.OR); + assertThat(tokens.get(2).type()).isEqualTo(TokenType.NOT); + assertThat(tokens.get(3).type()).isEqualTo(TokenType.IN); + assertThat(tokens.get(4).type()).isEqualTo(TokenType.MOTIF); + } + + @Test + public void testStringLiteral() { + List tokens = new Lexer("\"chess.com\"").tokenize(); + assertThat(tokens.get(0).type()).isEqualTo(TokenType.STRING); + assertThat(tokens.get(0).value()).isEqualTo("chess.com"); + } + + @Test + public void testAllOperators() { + List tokens = new Lexer("= != < <= > >=").tokenize(); + assertThat(tokens.get(0).type()).isEqualTo(TokenType.EQ); + assertThat(tokens.get(1).type()).isEqualTo(TokenType.NEQ); + assertThat(tokens.get(2).type()).isEqualTo(TokenType.LT); + assertThat(tokens.get(3).type()).isEqualTo(TokenType.LTE); + assertThat(tokens.get(4).type()).isEqualTo(TokenType.GT); + assertThat(tokens.get(5).type()).isEqualTo(TokenType.GTE); + } + + @Test + public void testDottedField() { + List tokens = new Lexer("white.elo").tokenize(); + assertThat(tokens.get(0).type()).isEqualTo(TokenType.IDENTIFIER); + assertThat(tokens.get(0).value()).isEqualTo("white"); + assertThat(tokens.get(1).type()).isEqualTo(TokenType.DOT); + assertThat(tokens.get(2).type()).isEqualTo(TokenType.IDENTIFIER); + assertThat(tokens.get(2).value()).isEqualTo("elo"); + } + + @Test + public void testInWithBrackets() { + List tokens = new Lexer("platform IN [\"a\", \"b\"]").tokenize(); + assertThat(tokens.get(0).type()).isEqualTo(TokenType.IDENTIFIER); + assertThat(tokens.get(1).type()).isEqualTo(TokenType.IN); + assertThat(tokens.get(2).type()).isEqualTo(TokenType.LBRACKET); + assertThat(tokens.get(3).type()).isEqualTo(TokenType.STRING); + assertThat(tokens.get(4).type()).isEqualTo(TokenType.COMMA); + assertThat(tokens.get(5).type()).isEqualTo(TokenType.STRING); + assertThat(tokens.get(6).type()).isEqualTo(TokenType.RBRACKET); + } + + @Test + public void testUnterminatedString() { + assertThatThrownBy(() -> new Lexer("\"unterminated").tokenize()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Unterminated string"); + } +} diff --git a/jvm/src/test/java/com/muchq/indexer/chessql/parser/BUILD.bazel b/jvm/src/test/java/com/muchq/indexer/chessql/parser/BUILD.bazel new file mode 100644 index 00000000..41baa9dc --- /dev/null +++ b/jvm/src/test/java/com/muchq/indexer/chessql/parser/BUILD.bazel @@ -0,0 +1,13 @@ +load("@contrib_rules_jvm//java:defs.bzl", "java_test_suite") + +java_test_suite( + name = "parser", + size = "small", + srcs = ["ParserTest.java"], + deps = [ + "//jvm/src/main/java/com/muchq/indexer/chessql/ast", + "//jvm/src/main/java/com/muchq/indexer/chessql/parser", + "@maven//:junit_junit", + "@maven//:org_assertj_assertj_core", + ], +) diff --git a/jvm/src/test/java/com/muchq/indexer/chessql/parser/ParserTest.java b/jvm/src/test/java/com/muchq/indexer/chessql/parser/ParserTest.java new file mode 100644 index 00000000..ef627cf2 --- /dev/null +++ b/jvm/src/test/java/com/muchq/indexer/chessql/parser/ParserTest.java @@ -0,0 +1,124 @@ +package com.muchq.indexer.chessql.parser; + +import com.muchq.indexer.chessql.ast.AndExpr; +import com.muchq.indexer.chessql.ast.ComparisonExpr; +import com.muchq.indexer.chessql.ast.Expr; +import com.muchq.indexer.chessql.ast.InExpr; +import com.muchq.indexer.chessql.ast.MotifExpr; +import com.muchq.indexer.chessql.ast.NotExpr; +import com.muchq.indexer.chessql.ast.OrExpr; +import org.junit.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class ParserTest { + + @Test + public void testSimpleComparison() { + Expr expr = Parser.parse("white_elo >= 2500"); + assertThat(expr).isInstanceOf(ComparisonExpr.class); + ComparisonExpr cmp = (ComparisonExpr) expr; + assertThat(cmp.field()).isEqualTo("white_elo"); + assertThat(cmp.operator()).isEqualTo(">="); + assertThat(cmp.value()).isEqualTo(2500); + } + + @Test + public void testDottedFieldComparison() { + Expr expr = Parser.parse("white.elo >= 2500"); + assertThat(expr).isInstanceOf(ComparisonExpr.class); + ComparisonExpr cmp = (ComparisonExpr) expr; + assertThat(cmp.field()).isEqualTo("white.elo"); + assertThat(cmp.operator()).isEqualTo(">="); + assertThat(cmp.value()).isEqualTo(2500); + } + + @Test + public void testMotifExpression() { + Expr expr = Parser.parse("motif(fork)"); + assertThat(expr).isInstanceOf(MotifExpr.class); + assertThat(((MotifExpr) expr).motifName()).isEqualTo("fork"); + } + + @Test + public void testAndExpression() { + Expr expr = Parser.parse("white.elo >= 2500 AND motif(cross_pin)"); + assertThat(expr).isInstanceOf(AndExpr.class); + AndExpr and = (AndExpr) expr; + assertThat(and.operands()).hasSize(2); + assertThat(and.operands().get(0)).isInstanceOf(ComparisonExpr.class); + assertThat(and.operands().get(1)).isInstanceOf(MotifExpr.class); + } + + @Test + public void testOrExpression() { + Expr expr = Parser.parse("motif(fork) OR motif(pin)"); + assertThat(expr).isInstanceOf(OrExpr.class); + OrExpr or = (OrExpr) expr; + assertThat(or.operands()).hasSize(2); + } + + @Test + public void testNotExpression() { + Expr expr = Parser.parse("NOT motif(pin)"); + assertThat(expr).isInstanceOf(NotExpr.class); + NotExpr not = (NotExpr) expr; + assertThat(not.operand()).isInstanceOf(MotifExpr.class); + } + + @Test + public void testInExpression() { + Expr expr = Parser.parse("platform IN [\"lichess\", \"chess.com\"]"); + assertThat(expr).isInstanceOf(InExpr.class); + InExpr in = (InExpr) expr; + assertThat(in.field()).isEqualTo("platform"); + assertThat(in.values()).isEqualTo(List.of("lichess", "chess.com")); + } + + @Test + public void testComplexExpression() { + Expr expr = Parser.parse("white.elo >= 2500 AND motif(fork) AND NOT motif(pin)"); + assertThat(expr).isInstanceOf(AndExpr.class); + AndExpr and = (AndExpr) expr; + assertThat(and.operands()).hasSize(3); + assertThat(and.operands().get(2)).isInstanceOf(NotExpr.class); + } + + @Test + public void testParenthesizedExpression() { + Expr expr = Parser.parse("(motif(fork) OR motif(pin)) AND white.elo > 2000"); + assertThat(expr).isInstanceOf(AndExpr.class); + AndExpr and = (AndExpr) expr; + assertThat(and.operands().get(0)).isInstanceOf(OrExpr.class); + assertThat(and.operands().get(1)).isInstanceOf(ComparisonExpr.class); + } + + @Test + public void testPrecedence() { + // AND binds tighter than OR + Expr expr = Parser.parse("motif(fork) OR motif(pin) AND white.elo > 2000"); + assertThat(expr).isInstanceOf(OrExpr.class); + OrExpr or = (OrExpr) expr; + assertThat(or.operands()).hasSize(2); + assertThat(or.operands().get(0)).isInstanceOf(MotifExpr.class); + assertThat(or.operands().get(1)).isInstanceOf(AndExpr.class); + } + + @Test + public void testStringComparison() { + Expr expr = Parser.parse("eco = \"B90\""); + assertThat(expr).isInstanceOf(ComparisonExpr.class); + ComparisonExpr cmp = (ComparisonExpr) expr; + assertThat(cmp.field()).isEqualTo("eco"); + assertThat(cmp.value()).isEqualTo("B90"); + } + + @Test + public void testParseError() { + assertThatThrownBy(() -> Parser.parse("AND")) + .isInstanceOf(ParseException.class); + } +} diff --git a/jvm/src/test/java/com/muchq/indexer/engine/BUILD.bazel b/jvm/src/test/java/com/muchq/indexer/engine/BUILD.bazel new file mode 100644 index 00000000..efdaf427 --- /dev/null +++ b/jvm/src/test/java/com/muchq/indexer/engine/BUILD.bazel @@ -0,0 +1,13 @@ +load("@contrib_rules_jvm//java:defs.bzl", "java_test_suite") + +java_test_suite( + name = "engine", + size = "small", + srcs = ["PgnParserTest.java"], + deps = [ + "//jvm/src/main/java/com/muchq/indexer/engine", + "//jvm/src/main/java/com/muchq/indexer/engine/model", + "@maven//:junit_junit", + "@maven//:org_assertj_assertj_core", + ], +) diff --git a/jvm/src/test/java/com/muchq/indexer/engine/PgnParserTest.java b/jvm/src/test/java/com/muchq/indexer/engine/PgnParserTest.java new file mode 100644 index 00000000..0f860409 --- /dev/null +++ b/jvm/src/test/java/com/muchq/indexer/engine/PgnParserTest.java @@ -0,0 +1,66 @@ +package com.muchq.indexer.engine; + +import com.muchq.indexer.engine.model.ParsedGame; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class PgnParserTest { + + private final PgnParser parser = new PgnParser(); + + @Test + public void testParseHeaders() { + String pgn = """ + [Event "Live Chess"] + [Site "Chess.com"] + [White "hikaru"] + [Black "magnuscarlsen"] + [Result "1-0"] + [ECO "B90"] + [WhiteElo "2850"] + [BlackElo "2830"] + + 1. e4 c5 2. Nf3 d6 3. d4 cxd4 4. Nxd4 Nf6 5. Nc3 a6 1-0 + """; + + ParsedGame game = parser.parse(pgn); + assertThat(game.headers()).containsEntry("Event", "Live Chess"); + assertThat(game.headers()).containsEntry("White", "hikaru"); + assertThat(game.headers()).containsEntry("Black", "magnuscarlsen"); + assertThat(game.headers()).containsEntry("ECO", "B90"); + assertThat(game.headers()).containsEntry("WhiteElo", "2850"); + } + + @Test + public void testParseMoveText() { + String pgn = """ + [Event "Test"] + + 1. e4 e5 2. Nf3 Nc6 3. Bb5 a6 1/2-1/2 + """; + + ParsedGame game = parser.parse(pgn); + assertThat(game.moveText()).contains("1. e4 e5"); + assertThat(game.moveText()).contains("Bb5"); + } + + @Test + public void testEmptyPgn() { + ParsedGame game = parser.parse(""); + assertThat(game.headers()).isEmpty(); + assertThat(game.moveText()).isEmpty(); + } + + @Test + public void testHeadersOnly() { + String pgn = """ + [Event "Test"] + [White "player1"] + """; + + ParsedGame game = parser.parse(pgn); + assertThat(game.headers()).hasSize(2); + assertThat(game.moveText()).isEmpty(); + } +} diff --git a/jvm/src/test/java/com/muchq/indexer/motifs/BUILD.bazel b/jvm/src/test/java/com/muchq/indexer/motifs/BUILD.bazel new file mode 100644 index 00000000..9c3f859f --- /dev/null +++ b/jvm/src/test/java/com/muchq/indexer/motifs/BUILD.bazel @@ -0,0 +1,17 @@ +load("@contrib_rules_jvm//java:defs.bzl", "java_test_suite") + +java_test_suite( + name = "motifs", + size = "small", + srcs = [ + "ForkDetectorTest.java", + "PinDetectorTest.java", + "SkewerDetectorTest.java", + ], + deps = [ + "//jvm/src/main/java/com/muchq/indexer/engine/model", + "//jvm/src/main/java/com/muchq/indexer/motifs", + "@maven//:junit_junit", + "@maven//:org_assertj_assertj_core", + ], +) diff --git a/jvm/src/test/java/com/muchq/indexer/motifs/ForkDetectorTest.java b/jvm/src/test/java/com/muchq/indexer/motifs/ForkDetectorTest.java new file mode 100644 index 00000000..5714df75 --- /dev/null +++ b/jvm/src/test/java/com/muchq/indexer/motifs/ForkDetectorTest.java @@ -0,0 +1,194 @@ +package com.muchq.indexer.motifs; + +import com.muchq.indexer.engine.model.GameFeatures; +import com.muchq.indexer.engine.model.Motif; +import com.muchq.indexer.engine.model.PositionContext; +import org.junit.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ForkDetectorTest { + + private final ForkDetector detector = new ForkDetector(); + + @Test + public void motifType() { + assertThat(detector.motif()).isEqualTo(Motif.FORK); + } + + // === Knight forks === + + @Test + public void knightFork_attacksKingAndQueen() { + // White knight on e6 forks black king on g7 and black queen on c7 + // Position: 8/2q3k1/4N3/8/8/8/8/4K3 w - - 0 1 + String fen = "8/2q3k1/4N3/8/8/8/8/4K3 w - - 0 1"; + List positions = List.of( + new PositionContext(10, fen, true) + ); + + // After white's move, it's black to move, so attacker is white (!whiteToMove) + // But ForkDetector checks for the side that just moved, which is white here + // Let's set whiteToMove=false to indicate black is to move (white just moved) + positions = List.of(new PositionContext(10, fen, false)); + + List occurrences = detector.detect(positions); + assertThat(occurrences).hasSize(1); + assertThat(occurrences.get(0).moveNumber()).isEqualTo(10); + } + + @Test + public void knightFork_attacksKingAndRook() { + // White knight on d5 forks black king on f6 and black rook on b4 + // Position: 8/8/5k2/3N4/1r6/8/8/4K3 b - - 0 1 + String fen = "8/8/5k2/3N4/1r6/8/8/4K3 b - - 0 1"; + List positions = List.of( + new PositionContext(15, fen, false) // black to move, white just forked + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).hasSize(1); + } + + @Test + public void knightFork_attacksQueenAndRook() { + // White knight on c6 forks black queen on e7 and black rook on a7 + String fen = "8/r3q3/2N5/8/8/8/8/4K2k b - - 0 1"; + List positions = List.of( + new PositionContext(12, fen, false) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).hasSize(1); + } + + @Test + public void knightFork_blackKnightForksWhitePieces() { + // Black knight on d4 forks white king on e2 and white queen on f5 + String fen = "8/8/8/5Q2/3n4/8/4K3/7k w - - 0 1"; + List positions = List.of( + new PositionContext(20, fen, true) // white to move, black just forked + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).hasSize(1); + } + + // === Pawn forks === + + @Test + public void pawnFork_whitePawnForksBlackPieces() { + // White pawn on d5 attacks black knights on c6 and e6 + String fen = "8/8/2n1n3/3P4/8/8/8/4K2k b - - 0 1"; + List positions = List.of( + new PositionContext(8, fen, false) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).hasSize(1); + } + + @Test + public void pawnFork_blackPawnForksWhitePieces() { + // Black pawn on e4 attacks white bishop on d3 and white knight on f3 + String fen = "7k/8/8/8/4p3/3B1N2/8/4K3 w - - 0 1"; + List positions = List.of( + new PositionContext(15, fen, true) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).hasSize(1); + } + + // === Queen forks === + + @Test + public void queenFork_attacksKingAndRook() { + // White queen on e5 forks black king on g7 and black rook on a5 + String fen = "8/6k1/8/r3Q3/8/8/8/4K3 b - - 0 1"; + List positions = List.of( + new PositionContext(25, fen, false) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).hasSize(1); + } + + // === Bishop forks === + + @Test + public void bishopFork_attacksTwoRooks() { + // White bishop on c4 attacks black rooks on a6 and f7 + String fen = "8/5r2/r7/8/2B5/8/8/4K2k b - - 0 1"; + List positions = List.of( + new PositionContext(18, fen, false) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).hasSize(1); + } + + // === No fork cases === + + @Test + public void noFork_onlyOnePieceAttacked() { + // White knight on e4 attacks only black queen on f6 + String fen = "8/8/5q2/8/4N3/8/8/4K2k b - - 0 1"; + List positions = List.of( + new PositionContext(10, fen, false) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).isEmpty(); + } + + @Test + public void noFork_attackingPawnsNotCounted() { + // White knight attacks two black pawns - pawns are value 1, not counted as valuable + String fen = "8/8/8/8/3N4/2p1p3/8/4K2k b - - 0 1"; + List positions = List.of( + new PositionContext(10, fen, false) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).isEmpty(); + } + + @Test + public void noFork_emptyPosition() { + // Starting position - no forks + String fen = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1"; + List positions = List.of( + new PositionContext(1, fen, true) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).isEmpty(); + } + + @Test + public void noFork_emptyList() { + List occurrences = detector.detect(List.of()); + assertThat(occurrences).isEmpty(); + } + + // === Multiple positions === + + @Test + public void multiplePositions_detectsForksInSome() { + List positions = List.of( + // No fork + new PositionContext(1, "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1", true), + // Fork: knight on e6 forks king g7 and queen c7 + new PositionContext(10, "8/2q3k1/4N3/8/8/8/8/4K3 b - - 0 1", false), + // No fork + new PositionContext(15, "8/8/8/8/8/8/8/4K2k w - - 0 1", true) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).hasSize(1); + assertThat(occurrences.get(0).moveNumber()).isEqualTo(10); + } +} diff --git a/jvm/src/test/java/com/muchq/indexer/motifs/PinDetectorTest.java b/jvm/src/test/java/com/muchq/indexer/motifs/PinDetectorTest.java new file mode 100644 index 00000000..73dc4854 --- /dev/null +++ b/jvm/src/test/java/com/muchq/indexer/motifs/PinDetectorTest.java @@ -0,0 +1,182 @@ +package com.muchq.indexer.motifs; + +import com.muchq.indexer.engine.model.GameFeatures; +import com.muchq.indexer.engine.model.Motif; +import com.muchq.indexer.engine.model.PositionContext; +import org.junit.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +public class PinDetectorTest { + + private final PinDetector detector = new PinDetector(); + + @Test + public void motifType() { + assertThat(detector.motif()).isEqualTo(Motif.PIN); + } + + // === Absolute pins (to king) === + + @Test + public void absolutePin_rookPinsKnightToKing() { + // Black rook on a4 pins white knight on e4 to white king on h4 + // Position: 8/8/8/8/r3N2K/8/8/7k w - - 0 1 + String fen = "8/8/8/8/r3N2K/8/8/7k w - - 0 1"; + List positions = List.of( + new PositionContext(15, fen, true) // white to move, knight is pinned + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).hasSize(1); + } + + @Test + public void absolutePin_bishopPinsBishopToKing() { + // Black bishop on a1 pins white bishop on d4 to white king on g7 + String fen = "8/6K1/8/8/3B4/8/8/b6k w - - 0 1"; + List positions = List.of( + new PositionContext(20, fen, true) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).hasSize(1); + } + + @Test + public void absolutePin_queenPinsRookToKing() { + // Black queen on a8 pins white rook on d8 to white king on h8 + String fen = "q2R3K/8/8/8/8/8/8/7k w - - 0 1"; + List positions = List.of( + new PositionContext(25, fen, true) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).hasSize(1); + } + + @Test + public void absolutePin_blackPieceIsPinned() { + // White rook on a5 pins black knight on e5 to black king on h5 + String fen = "8/8/8/R3n2k/8/8/8/4K3 b - - 0 1"; + List positions = List.of( + new PositionContext(18, fen, false) // black to move, knight is pinned + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).hasSize(1); + } + + @Test + public void absolutePin_diagonalPin() { + // White bishop on b1 pins black knight on d3 to black king on f5 + String fen = "8/8/8/5k2/8/3n4/8/1B2K3 b - - 0 1"; + List positions = List.of( + new PositionContext(12, fen, false) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).hasSize(1); + } + + // === No pin cases === + + @Test + public void noPin_startingPosition() { + String fen = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1"; + List positions = List.of( + new PositionContext(1, fen, true) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).isEmpty(); + } + + @Test + public void noPin_noPieceBetweenAttackerAndKing() { + // Black rook attacks white king directly, no pin + String fen = "8/8/8/8/r6K/8/8/7k w - - 0 1"; + List positions = List.of( + new PositionContext(10, fen, true) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).isEmpty(); + } + + @Test + public void noPin_twoPiecesBetweenAttackerAndKing() { + // Black rook, two white pieces, white king - not a pin + String fen = "8/8/8/8/r2NN2K/8/8/7k w - - 0 1"; + List positions = List.of( + new PositionContext(10, fen, true) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).isEmpty(); + } + + @Test + public void noPin_knightCannotPin() { + // Knights cannot create pins (don't slide) + String fen = "8/8/2n5/8/3B4/8/8/4K2k w - - 0 1"; + List positions = List.of( + new PositionContext(10, fen, true) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).isEmpty(); + } + + @Test + public void noPin_wrongPieceType() { + // Rook on diagonal cannot pin (rooks don't attack diagonally) + String fen = "8/6K1/8/8/3B4/8/8/r6k w - - 0 1"; + List positions = List.of( + new PositionContext(10, fen, true) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).isEmpty(); + } + + @Test + public void noPin_emptyPositionList() { + List occurrences = detector.detect(List.of()); + assertThat(occurrences).isEmpty(); + } + + // === Helper methods === + + @Test + public void parsePlacement_startingPosition() { + String placement = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR"; + int[][] board = PinDetector.parsePlacement(placement); + + // Check some squares + assertThat(board[0][0]).isEqualTo(-4); // black rook a8 + assertThat(board[0][4]).isEqualTo(-6); // black king e8 + assertThat(board[7][4]).isEqualTo(6); // white king e1 + assertThat(board[7][3]).isEqualTo(5); // white queen d1 + assertThat(board[4][4]).isEqualTo(0); // empty e4 + } + + @Test + public void pieceValue_allPieces() { + assertThat(PinDetector.pieceValue('K')).isEqualTo(6); + assertThat(PinDetector.pieceValue('Q')).isEqualTo(5); + assertThat(PinDetector.pieceValue('R')).isEqualTo(4); + assertThat(PinDetector.pieceValue('B')).isEqualTo(3); + assertThat(PinDetector.pieceValue('N')).isEqualTo(2); + assertThat(PinDetector.pieceValue('P')).isEqualTo(1); + assertThat(PinDetector.pieceValue('k')).isEqualTo(-6); + assertThat(PinDetector.pieceValue('q')).isEqualTo(-5); + assertThat(PinDetector.pieceValue('r')).isEqualTo(-4); + assertThat(PinDetector.pieceValue('b')).isEqualTo(-3); + assertThat(PinDetector.pieceValue('n')).isEqualTo(-2); + assertThat(PinDetector.pieceValue('p')).isEqualTo(-1); + assertThat(PinDetector.pieceValue('x')).isEqualTo(0); // invalid + } +} diff --git a/jvm/src/test/java/com/muchq/indexer/motifs/SkewerDetectorTest.java b/jvm/src/test/java/com/muchq/indexer/motifs/SkewerDetectorTest.java new file mode 100644 index 00000000..a9163180 --- /dev/null +++ b/jvm/src/test/java/com/muchq/indexer/motifs/SkewerDetectorTest.java @@ -0,0 +1,178 @@ +package com.muchq.indexer.motifs; + +import com.muchq.indexer.engine.model.GameFeatures; +import com.muchq.indexer.engine.model.Motif; +import com.muchq.indexer.engine.model.PositionContext; +import org.junit.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +public class SkewerDetectorTest { + + private final SkewerDetector detector = new SkewerDetector(); + + @Test + public void motifType() { + assertThat(detector.motif()).isEqualTo(Motif.SKEWER); + } + + // === Skewer cases === + // A skewer is when a more valuable piece is in front and a less valuable piece is behind + + @Test + public void skewer_rookSkewersKingAndRook() { + // White rook on a4 attacks black king on e4, with black rook on h4 behind + // King (6) > Rook (4), so this is a skewer + String fen = "8/8/8/8/R3k2r/8/8/4K3 b - - 0 1"; + List positions = List.of( + new PositionContext(15, fen, false) // black to move, white just skewered + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).hasSize(1); + } + + @Test + public void skewer_queenSkewersKingAndBishop() { + // White queen on a1 attacks black king on d4, with black bishop on g7 behind + String fen = "8/6b1/8/8/3k4/8/8/Q3K3 b - - 0 1"; + List positions = List.of( + new PositionContext(20, fen, false) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).hasSize(1); + } + + @Test + public void skewer_bishopSkewersQueenAndRook() { + // White bishop on b1 attacks black queen on d3, with black rook on f5 behind + // Queen (5) > Rook (4), so this is a skewer + String fen = "8/8/8/5r2/8/3q4/8/1B2K2k b - - 0 1"; + List positions = List.of( + new PositionContext(18, fen, false) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).hasSize(1); + } + + @Test + public void skewer_rookSkewersQueenAndKnight() { + // White rook on a5 attacks black queen on d5, with black knight on h5 behind + String fen = "8/8/8/R2q3n/8/8/8/4K2k b - - 0 1"; + List positions = List.of( + new PositionContext(22, fen, false) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).hasSize(1); + } + + @Test + public void skewer_blackSkewersWhitePieces() { + // Black rook on h4 attacks white king on e4, with white bishop on b4 behind + String fen = "8/8/8/8/1B2K2r/8/8/7k w - - 0 1"; + List positions = List.of( + new PositionContext(15, fen, true) // white to move, black just skewered + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).hasSize(1); + } + + // === Not a skewer cases === + + @Test + public void notSkewer_lessValuableInFront() { + // This is a PIN, not a skewer: knight in front, king behind + // White rook attacks black knight on e4, with black king on h4 behind + String fen = "8/8/8/8/R3n2k/8/8/4K3 b - - 0 1"; + List positions = List.of( + new PositionContext(10, fen, false) + ); + + // Should NOT detect skewer (this would be detected as a pin) + List occurrences = detector.detect(positions); + assertThat(occurrences).isEmpty(); + } + + @Test + public void notSkewer_sameValuePieces() { + // Rook attacks rook with rook behind - same value, not a skewer + String fen = "8/8/8/8/R3r2r/8/8/4K2k b - - 0 1"; + List positions = List.of( + new PositionContext(10, fen, false) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).isEmpty(); + } + + @Test + public void notSkewer_onlyOnePieceOnRay() { + // Rook attacks king, but nothing behind + String fen = "8/8/8/8/R3k3/8/8/4K3 b - - 0 1"; + List positions = List.of( + new PositionContext(10, fen, false) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).isEmpty(); + } + + @Test + public void notSkewer_pawnBehind() { + // King skewered to pawn - but pawn value (1) < 2, not counted + String fen = "8/8/8/8/R3k2p/8/8/4K3 b - - 0 1"; + List positions = List.of( + new PositionContext(10, fen, false) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).isEmpty(); + } + + @Test + public void notSkewer_friendlyPieceBlocks() { + // White rook on a4, white knight on c4 blocks any skewer potential + String fen = "8/8/8/8/R1N1k2r/8/8/4K3 b - - 0 1"; + List positions = List.of( + new PositionContext(10, fen, false) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).isEmpty(); + } + + @Test + public void notSkewer_knightCannotSkewer() { + // Knights cannot create skewers (don't slide) + String fen = "8/8/5q2/8/4N3/8/3r4/4K2k b - - 0 1"; + List positions = List.of( + new PositionContext(10, fen, false) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).isEmpty(); + } + + @Test + public void notSkewer_startingPosition() { + String fen = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1"; + List positions = List.of( + new PositionContext(1, fen, true) + ); + + List occurrences = detector.detect(positions); + assertThat(occurrences).isEmpty(); + } + + @Test + public void notSkewer_emptyList() { + List occurrences = detector.detect(List.of()); + assertThat(occurrences).isEmpty(); + } +} diff --git a/jvm/src/test/java/com/muchq/indexer/queue/BUILD.bazel b/jvm/src/test/java/com/muchq/indexer/queue/BUILD.bazel new file mode 100644 index 00000000..cada7e43 --- /dev/null +++ b/jvm/src/test/java/com/muchq/indexer/queue/BUILD.bazel @@ -0,0 +1,12 @@ +load("@contrib_rules_jvm//java:defs.bzl", "java_test_suite") + +java_test_suite( + name = "queue", + size = "small", + srcs = ["InMemoryIndexQueueTest.java"], + deps = [ + "//jvm/src/main/java/com/muchq/indexer/queue", + "@maven//:junit_junit", + "@maven//:org_assertj_assertj_core", + ], +) diff --git a/jvm/src/test/java/com/muchq/indexer/queue/InMemoryIndexQueueTest.java b/jvm/src/test/java/com/muchq/indexer/queue/InMemoryIndexQueueTest.java new file mode 100644 index 00000000..e7e26ddf --- /dev/null +++ b/jvm/src/test/java/com/muchq/indexer/queue/InMemoryIndexQueueTest.java @@ -0,0 +1,43 @@ +package com.muchq.indexer.queue; + +import org.junit.Test; + +import java.time.Duration; +import java.util.Optional; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class InMemoryIndexQueueTest { + + @Test + public void testEnqueueAndPoll() { + InMemoryIndexQueue queue = new InMemoryIndexQueue(); + IndexMessage message = new IndexMessage(UUID.randomUUID(), "hikaru", "chess.com", "2024-01", "2024-01"); + + queue.enqueue(message); + assertThat(queue.size()).isEqualTo(1); + + Optional polled = queue.poll(Duration.ofMillis(100)); + assertThat(polled).isPresent(); + assertThat(polled.get().player()).isEqualTo("hikaru"); + assertThat(queue.size()).isEqualTo(0); + } + + @Test + public void testPollEmpty() { + InMemoryIndexQueue queue = new InMemoryIndexQueue(); + Optional polled = queue.poll(Duration.ofMillis(50)); + assertThat(polled).isEmpty(); + } + + @Test + public void testSize() { + InMemoryIndexQueue queue = new InMemoryIndexQueue(); + assertThat(queue.size()).isEqualTo(0); + + queue.enqueue(new IndexMessage(UUID.randomUUID(), "a", "chess.com", "2024-01", "2024-01")); + queue.enqueue(new IndexMessage(UUID.randomUUID(), "b", "chess.com", "2024-01", "2024-01")); + assertThat(queue.size()).isEqualTo(2); + } +} diff --git a/jvm/src/test/java/com/muchq/indexer/worker/BUILD.bazel b/jvm/src/test/java/com/muchq/indexer/worker/BUILD.bazel new file mode 100644 index 00000000..1c8f94b2 --- /dev/null +++ b/jvm/src/test/java/com/muchq/indexer/worker/BUILD.bazel @@ -0,0 +1,12 @@ +load("@contrib_rules_jvm//java:defs.bzl", "java_test_suite") + +java_test_suite( + name = "worker", + size = "small", + srcs = ["ResultMapperTest.java"], + deps = [ + "//jvm/src/main/java/com/muchq/indexer/worker", + "@maven//:junit_junit", + "@maven//:org_assertj_assertj_core", + ], +) diff --git a/jvm/src/test/java/com/muchq/indexer/worker/ResultMapperTest.java b/jvm/src/test/java/com/muchq/indexer/worker/ResultMapperTest.java new file mode 100644 index 00000000..8383dc1e --- /dev/null +++ b/jvm/src/test/java/com/muchq/indexer/worker/ResultMapperTest.java @@ -0,0 +1,184 @@ +package com.muchq.indexer.worker; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ResultMapperTest { + + // === White wins === + + @Test + public void whiteWins_explicitWin() { + assertThat(ResultMapper.mapResult("win", "resigned")).isEqualTo("1-0"); + assertThat(ResultMapper.mapResult("win", "checkmated")).isEqualTo("1-0"); + assertThat(ResultMapper.mapResult("win", "timeout")).isEqualTo("1-0"); + } + + @Test + public void whiteWins_blackResigned() { + assertThat(ResultMapper.mapResult(null, "resigned")).isEqualTo("1-0"); + } + + @Test + public void whiteWins_blackCheckmated() { + assertThat(ResultMapper.mapResult(null, "checkmated")).isEqualTo("1-0"); + } + + @Test + public void whiteWins_blackTimeout() { + assertThat(ResultMapper.mapResult(null, "timeout")).isEqualTo("1-0"); + } + + @Test + public void whiteWins_blackAbandoned() { + assertThat(ResultMapper.mapResult(null, "abandoned")).isEqualTo("1-0"); + } + + @Test + public void whiteWins_blackLose() { + assertThat(ResultMapper.mapResult(null, "lose")).isEqualTo("1-0"); + } + + // === Black wins === + + @Test + public void blackWins_explicitWin() { + assertThat(ResultMapper.mapResult("resigned", "win")).isEqualTo("0-1"); + assertThat(ResultMapper.mapResult("checkmated", "win")).isEqualTo("0-1"); + assertThat(ResultMapper.mapResult("timeout", "win")).isEqualTo("0-1"); + } + + @Test + public void blackWins_whiteResigned() { + assertThat(ResultMapper.mapResult("resigned", null)).isEqualTo("0-1"); + } + + @Test + public void blackWins_whiteCheckmated() { + assertThat(ResultMapper.mapResult("checkmated", null)).isEqualTo("0-1"); + } + + @Test + public void blackWins_whiteTimeout() { + assertThat(ResultMapper.mapResult("timeout", null)).isEqualTo("0-1"); + } + + @Test + public void blackWins_whiteAbandoned() { + assertThat(ResultMapper.mapResult("abandoned", null)).isEqualTo("0-1"); + } + + @Test + public void blackWins_whiteLose() { + assertThat(ResultMapper.mapResult("lose", null)).isEqualTo("0-1"); + } + + // === Draws === + + @Test + public void draw_agreed() { + assertThat(ResultMapper.mapResult("agreed", "agreed")).isEqualTo("1/2-1/2"); + assertThat(ResultMapper.mapResult("agreed", null)).isEqualTo("1/2-1/2"); + assertThat(ResultMapper.mapResult(null, "agreed")).isEqualTo("1/2-1/2"); + } + + @Test + public void draw_repetition() { + assertThat(ResultMapper.mapResult("repetition", "repetition")).isEqualTo("1/2-1/2"); + assertThat(ResultMapper.mapResult("repetition", null)).isEqualTo("1/2-1/2"); + } + + @Test + public void draw_stalemate() { + assertThat(ResultMapper.mapResult("stalemate", "stalemate")).isEqualTo("1/2-1/2"); + } + + @Test + public void draw_insufficient() { + assertThat(ResultMapper.mapResult("insufficient", "insufficient")).isEqualTo("1/2-1/2"); + } + + @Test + public void draw_50move() { + assertThat(ResultMapper.mapResult("50move", "50move")).isEqualTo("1/2-1/2"); + } + + @Test + public void draw_timevsinsufficient() { + assertThat(ResultMapper.mapResult("timevsinsufficient", "timevsinsufficient")).isEqualTo("1/2-1/2"); + } + + @Test + public void draw_drawn() { + assertThat(ResultMapper.mapResult("drawn", null)).isEqualTo("1/2-1/2"); + } + + // === Unknown === + + @Test + public void unknown_bothNull() { + assertThat(ResultMapper.mapResult(null, null)).isEqualTo("unknown"); + } + + @Test + public void unknown_unrecognizedValues() { + assertThat(ResultMapper.mapResult("something", "else")).isEqualTo("unknown"); + } + + // === isDrawResult === + + @Test + public void isDrawResult_recognizesAllDrawTypes() { + assertThat(ResultMapper.isDrawResult("agreed")).isTrue(); + assertThat(ResultMapper.isDrawResult("repetition")).isTrue(); + assertThat(ResultMapper.isDrawResult("stalemate")).isTrue(); + assertThat(ResultMapper.isDrawResult("insufficient")).isTrue(); + assertThat(ResultMapper.isDrawResult("50move")).isTrue(); + assertThat(ResultMapper.isDrawResult("timevsinsufficient")).isTrue(); + assertThat(ResultMapper.isDrawResult("drawn")).isTrue(); + } + + @Test + public void isDrawResult_rejectsNonDraws() { + assertThat(ResultMapper.isDrawResult("win")).isFalse(); + assertThat(ResultMapper.isDrawResult("resigned")).isFalse(); + assertThat(ResultMapper.isDrawResult("checkmated")).isFalse(); + assertThat(ResultMapper.isDrawResult(null)).isFalse(); + } + + // === isLossResult === + + @Test + public void isLossResult_recognizesAllLossTypes() { + assertThat(ResultMapper.isLossResult("resigned")).isTrue(); + assertThat(ResultMapper.isLossResult("checkmated")).isTrue(); + assertThat(ResultMapper.isLossResult("timeout")).isTrue(); + assertThat(ResultMapper.isLossResult("abandoned")).isTrue(); + assertThat(ResultMapper.isLossResult("lose")).isTrue(); + } + + @Test + public void isLossResult_rejectsNonLosses() { + assertThat(ResultMapper.isLossResult("win")).isFalse(); + assertThat(ResultMapper.isLossResult("agreed")).isFalse(); + assertThat(ResultMapper.isLossResult("repetition")).isFalse(); + assertThat(ResultMapper.isLossResult(null)).isFalse(); + } + + // === Edge cases === + + @Test + public void explicitWinTakesPrecedenceOverLoss() { + // If both sides have a result, "win" should be authoritative + assertThat(ResultMapper.mapResult("win", "resigned")).isEqualTo("1-0"); + assertThat(ResultMapper.mapResult("resigned", "win")).isEqualTo("0-1"); + } + + @Test + public void drawTakesPrecedenceOverUnknown() { + // A draw result on either side should yield 1/2-1/2 + assertThat(ResultMapper.mapResult("repetition", "unknown_value")).isEqualTo("1/2-1/2"); + assertThat(ResultMapper.mapResult("unknown_value", "stalemate")).isEqualTo("1/2-1/2"); + } +} diff --git a/maven_install.json b/maven_install.json index 718f506e..ff161019 100755 --- a/maven_install.json +++ b/maven_install.json @@ -1,7 +1,7 @@ { "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": "THERE_IS_NO_DATA_ONLY_ZUUL", - "__INPUT_ARTIFACTS_HASH": -285956844, - "__RESOLVED_ARTIFACTS_HASH": 1666541772, + "__INPUT_ARTIFACTS_HASH": -82233566, + "__RESOLVED_ARTIFACTS_HASH": 1853180561, "conflict_resolution": { "com.fasterxml.jackson.core:jackson-annotations": "com.fasterxml.jackson.core:jackson-annotations:2.19.2", "com.fasterxml.jackson.core:jackson-core": "com.fasterxml.jackson.core:jackson-core:2.19.2", @@ -137,12 +137,30 @@ }, "version": "3.1" }, + "com.h2database:h2": { + "shasums": { + "jar": "b9d8f19358ada82a4f6eb5b174c6cfe320a375b5a9cb5a4fe456d623e6e55497" + }, + "version": "2.2.224" + }, "com.thoughtworks.paranamer:paranamer": { "shasums": { "jar": "a9df136f2e926b37a838a5b4e2227343c3a755d15724b3a8350b4aea4b158945" }, "version": "2.8.3" }, + "com.zaxxer:HikariCP": { + "shasums": { + "jar": "709f378c05756280939ce50fc1b1f1a53bb8e1899dc1b249f21f12703640b48b" + }, + "version": "6.3.3" + }, + "io.github.tors42:chariot": { + "shasums": { + "jar": "1479fa9edc5bb61dab644e70ba5c1552044d09c3f7ae48e6c6d6ed0cbbbfa828" + }, + "version": "0.1.10" + }, "io.micronaut.jaxrs:micronaut-jaxrs-common": { "shasums": { "jar": "983c901843359b4e23d087c60d24e02214c492434a92af2fd928940f9818bea1" @@ -617,6 +635,12 @@ }, "version": "9.8" }, + "org.postgresql:postgresql": { + "shasums": { + "jar": "188976721ead8e8627eb6d8389d500dccc0c9bebd885268a3047180274a6031e" + }, + "version": "42.7.4" + }, "org.reactivestreams:reactive-streams": { "shasums": { "jar": "f75ca597789b3dac58f61857b9ac2e1034a68fa672db35055a8fb4509e325f28" @@ -678,6 +702,9 @@ "com.google.j2objc:j2objc-annotations", "org.jspecify:jspecify" ], + "com.zaxxer:HikariCP": [ + "org.slf4j:slf4j-api" + ], "io.micronaut.jaxrs:micronaut-jaxrs-common": [ "io.micronaut:micronaut-http", "io.micronaut:micronaut-inject", @@ -1045,6 +1072,9 @@ "org.ow2.asm:asm", "org.ow2.asm:asm-analysis", "org.ow2.asm:asm-tree" + ], + "org.postgresql:postgresql": [ + "org.checkerframework:checker-qual" ] }, "packages": { @@ -1304,9 +1334,89 @@ "com.google.j2objc:j2objc-annotations": [ "com.google.j2objc.annotations" ], + "com.h2database:h2": [ + "org.h2", + "org.h2.api", + "org.h2.bnf", + "org.h2.bnf.context", + "org.h2.command", + "org.h2.command.ddl", + "org.h2.command.dml", + "org.h2.command.query", + "org.h2.compress", + "org.h2.constraint", + "org.h2.engine", + "org.h2.expression", + "org.h2.expression.aggregate", + "org.h2.expression.analysis", + "org.h2.expression.condition", + "org.h2.expression.function", + "org.h2.expression.function.table", + "org.h2.fulltext", + "org.h2.index", + "org.h2.jdbc", + "org.h2.jdbc.meta", + "org.h2.jdbcx", + "org.h2.jmx", + "org.h2.message", + "org.h2.mode", + "org.h2.mvstore", + "org.h2.mvstore.cache", + "org.h2.mvstore.db", + "org.h2.mvstore.rtree", + "org.h2.mvstore.tx", + "org.h2.mvstore.type", + "org.h2.result", + "org.h2.schema", + "org.h2.security", + "org.h2.security.auth", + "org.h2.security.auth.impl", + "org.h2.server", + "org.h2.server.pg", + "org.h2.server.web", + "org.h2.store", + "org.h2.store.fs", + "org.h2.store.fs.async", + "org.h2.store.fs.disk", + "org.h2.store.fs.encrypt", + "org.h2.store.fs.mem", + "org.h2.store.fs.niomapped", + "org.h2.store.fs.niomem", + "org.h2.store.fs.rec", + "org.h2.store.fs.retry", + "org.h2.store.fs.split", + "org.h2.store.fs.zip", + "org.h2.table", + "org.h2.tools", + "org.h2.util", + "org.h2.util.geometry", + "org.h2.util.json", + "org.h2.value", + "org.h2.value.lob" + ], "com.thoughtworks.paranamer:paranamer": [ "com.thoughtworks.paranamer" ], + "com.zaxxer:HikariCP": [ + "com.zaxxer.hikari", + "com.zaxxer.hikari.hibernate", + "com.zaxxer.hikari.metrics", + "com.zaxxer.hikari.metrics.dropwizard", + "com.zaxxer.hikari.metrics.micrometer", + "com.zaxxer.hikari.metrics.prometheus", + "com.zaxxer.hikari.pool", + "com.zaxxer.hikari.util" + ], + "io.github.tors42:chariot": [ + "chariot", + "chariot.api", + "chariot.internal", + "chariot.internal.impl", + "chariot.internal.modeladapter", + "chariot.internal.yayson", + "chariot.model", + "chariot.util" + ], "io.micronaut.jaxrs:micronaut-jaxrs-common": [ "io.micronaut.jaxrs.common", "io.micronaut.jaxrs.common.body.standard" @@ -2079,6 +2189,46 @@ "org.ow2.asm:asm-util": [ "org.objectweb.asm.util" ], + "org.postgresql:postgresql": [ + "org.postgresql", + "org.postgresql.copy", + "org.postgresql.core", + "org.postgresql.core.v3", + "org.postgresql.core.v3.adaptivefetch", + "org.postgresql.core.v3.replication", + "org.postgresql.ds", + "org.postgresql.ds.common", + "org.postgresql.fastpath", + "org.postgresql.geometric", + "org.postgresql.gss", + "org.postgresql.hostchooser", + "org.postgresql.jdbc", + "org.postgresql.jdbc2", + "org.postgresql.jdbc2.optional", + "org.postgresql.jdbc3", + "org.postgresql.jdbcurlresolver", + "org.postgresql.largeobject", + "org.postgresql.osgi", + "org.postgresql.plugin", + "org.postgresql.replication", + "org.postgresql.replication.fluent", + "org.postgresql.replication.fluent.logical", + "org.postgresql.replication.fluent.physical", + "org.postgresql.shaded.com.ongres.saslprep", + "org.postgresql.shaded.com.ongres.scram.client", + "org.postgresql.shaded.com.ongres.scram.common", + "org.postgresql.shaded.com.ongres.scram.common.exception", + "org.postgresql.shaded.com.ongres.scram.common.util", + "org.postgresql.shaded.com.ongres.stringprep", + "org.postgresql.ssl", + "org.postgresql.ssl.jdbc4", + "org.postgresql.sspi", + "org.postgresql.translation", + "org.postgresql.util", + "org.postgresql.util.internal", + "org.postgresql.xa", + "org.postgresql.xml" + ], "org.reactivestreams:reactive-streams": [ "org.reactivestreams" ], @@ -2142,7 +2292,10 @@ "com.google.guava:guava", "com.google.guava:listenablefuture", "com.google.j2objc:j2objc-annotations", + "com.h2database:h2", "com.thoughtworks.paranamer:paranamer", + "com.zaxxer:HikariCP", + "io.github.tors42:chariot", "io.micronaut.jaxrs:micronaut-jaxrs-common", "io.micronaut.jaxrs:micronaut-jaxrs-processor", "io.micronaut.jaxrs:micronaut-jaxrs-server", @@ -2222,6 +2375,7 @@ "org.ow2.asm:asm-commons", "org.ow2.asm:asm-tree", "org.ow2.asm:asm-util", + "org.postgresql:postgresql", "org.reactivestreams:reactive-streams", "org.scala-lang:scala-library", "org.slf4j:slf4j-api" @@ -2266,6 +2420,11 @@ "com.fasterxml.jackson.module.scala.DefaultScalaModule" ] }, + "com.h2database:h2": { + "java.sql.Driver": [ + "org.h2.Driver" + ] + }, "io.micronaut.jaxrs:micronaut-jaxrs-common": { "io.micronaut.core.convert.TypeConverterRegistrar": [ "io.micronaut.jaxrs.common.JaxRsConverterRegistrar" @@ -2573,6 +2732,11 @@ "org.eclipse.jetty.webapp.WebInfConfiguration", "org.eclipse.jetty.webapp.WebXmlConfiguration" ] + }, + "org.postgresql:postgresql": { + "java.sql.Driver": [ + "org.postgresql.Driver" + ] } }, "version": "2"