From 09590bf8aa371698716f565a25c7f34a52046f90 Mon Sep 17 00:00:00 2001 From: John Hernley Date: Fri, 7 Feb 2025 20:05:38 +0000 Subject: [PATCH 1/3] implement follow and auto-follow --- lib/elastomer_client/client/ccr.rb | 64 ++++++++++++++++++++++++++++++ script/setup-ccr | 2 +- test/client/ccr_test.rb | 54 +++++++++++++++++++++++++ test/test_helper.rb | 13 ++++++ 4 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 lib/elastomer_client/client/ccr.rb create mode 100644 test/client/ccr_test.rb diff --git a/lib/elastomer_client/client/ccr.rb b/lib/elastomer_client/client/ccr.rb new file mode 100644 index 00000000..869efe43 --- /dev/null +++ b/lib/elastomer_client/client/ccr.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module ElastomerClient + class Client + + # Returns a CCR instance + def ccr + Ccr.new(self) + end + + class Ccr + # Create a new Ccr for initiating requests for cross cluster replication. + # More context: https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-apis.html + # + # client - ElastomerClient::Client used for HTTP requests to the server + def initialize(client) + @client = client + end + + attr_reader :client + + # Creates a new follower index for the provided leader index on the remote cluster. + # + # follower_index - String name of the follower index to create + # body - Hash of the request body + # :remote_cluster - String name of the remote cluster. Required. + # :leader_index - String name of the leader index. Required. + # params - Hash of query parameters + # + # Examples + # + # ccr.follow("follower_index", { leader_index: "leader_index", remote_cluster: "leader" }) + # + # See https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-put-follow.html + def follow(follower_index, body, params={}) + response = client.put "/#{follower_index}/_ccr/follow", params.merge(params, body:, action: "follow", rest_api: "ccr") + response.body + end + + # Creates a new auto-follow pattern for the provided remote cluster. + # + # pattern_name - String name of the auto-follow pattern to create + # body - Hash of the request body + # :remote_cluster - String name of the remote cluster. Required. + # :leader_index_patterns - An array of simple index patterns to match against indices in the remote cluster + # :leader_index_exclusion_patterns - An array of simple index patterns that can be used to exclude indices from being auto-followed. + # :follow_index_pattern - The name of follower index. The template {{leader_index}} can be used to derive + # the name of the follower index from the name of the leader index. + # params - Hash of query parameters + # + # Examples + # + # ccr.auto_follow("follower_pattern", { remote_cluster: "leader", leader_index_patterns: ["leader_index*"] + # follow_index_pattern: "{{leader_index}}-follower" }) + # + # See https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-put-auto-follow-pattern.html + def auto_follow(pattern_name, body, params={}) + response = client.put "/_ccr/auto_follow/#{pattern_name}", params.merge(params, body:, action: "create_auto_follow_pattern", rest_api: "ccr") + response.body + end + + end + end +end diff --git a/script/setup-ccr b/script/setup-ccr index 3f721b05..140dec5c 100755 --- a/script/setup-ccr +++ b/script/setup-ccr @@ -65,7 +65,7 @@ case "$1" in "persistent": { "cluster": { "remote": { - "cluster_one": { + "leader": { "seeds": ["es8.13:9300"] } } diff --git a/test/client/ccr_test.rb b/test/client/ccr_test.rb new file mode 100644 index 00000000..e76b6de4 --- /dev/null +++ b/test/client/ccr_test.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require_relative "../test_helper" + +describe ElastomerClient::Client::Ccr do + before do + skip "Cannot test Ccr API without a replica cluster" unless $replica_client.available? + + @leader_index = $client.index("leader_index") + @follower_index = $replica_client.index("follower_index") + if @leader_index.exists? + @leader_index.delete + end + if @follower_index.exists? + @follower_index.delete + end + @leader_index.create(default_index_settings) + wait_for_index(@leader_index.name, "green") + + @leader_index.docs.index(document_wrapper("book", { _id: 1, title: "Book 1" })) + @leader_index.refresh + end + + after do + @leader_index.delete if @leader_index.exists? + @follower_index.delete if @follower_index.exists? + end + + it "successfully follows a leader index" do + ccr = $replica_client.ccr + + ccr.follow(@follower_index.name, { leader_index: @leader_index.name, remote_cluster: "leader" }) + wait_for_index(@follower_index.name, "green") + doc = @follower_index.docs.get(id: 1, type: "book") + + assert_equal "Book 1", doc["_source"]["title"] + end + + it "successfully implements an auto-follow policy" do + ccr = $replica_client.ccr + + ccr.auto_follow("follower_pattern", { remote_cluster: "leader", leader_index_patterns: ["*"], follow_index_pattern: "{{leader_index}}-follower" }) + + index = $client.index("followed_index") + index.create(default_index_settings) + wait_for_index(index.name, "green") + follower_index = $replica_client.index("followed_index-follower") + wait_for_index(follower_index.name, "green") + assert follower_index.exists? + + index.delete + follower_index.delete + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb index 604ea16a..586dc004 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -46,6 +46,19 @@ puts "Elasticsearch version is #{$client.version}" +# Create client instance for replica cluster +$replica_client_params = { + port: ENV.fetch("ES_REPLICA_PORT", 9201), + read_timeout: 10, + open_timeout: 1, + opaque_id: false, + strict_params: true, + compress_body: true +} +$replica_client = ElastomerClient::Client.new(**$replica_client_params) + +puts "Replica server is unavailable at #{$replica_client.url}" unless $replica_client.available? + # remove any lingering test indices from the cluster Minitest.after_run do $client.cluster.indices.keys.each do |name| From 8a50763f9a455c3c2e64336d45a7050c2397d863 Mon Sep 17 00:00:00 2001 From: John Hernley Date: Mon, 10 Feb 2025 17:46:16 +0000 Subject: [PATCH 2/3] update docs and comment out auto-follow --- README.md | 5 +++++ lib/elastomer_client/client/ccr.rb | 10 +++++----- test/client/ccr_test.rb | 26 +++++++++++++------------- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 4d1c94b3..eb89fb0d 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,11 @@ ES 8 ES_PORT=9208 rake test ``` +CCR tests: +``` +ES_PORT=9208 ES_REPLICA_PORT=9209 rake test +``` + ES 5 ``` ES_PORT=9205 rake test diff --git a/lib/elastomer_client/client/ccr.rb b/lib/elastomer_client/client/ccr.rb index 869efe43..5e6ba238 100644 --- a/lib/elastomer_client/client/ccr.rb +++ b/lib/elastomer_client/client/ccr.rb @@ -32,7 +32,7 @@ def initialize(client) # ccr.follow("follower_index", { leader_index: "leader_index", remote_cluster: "leader" }) # # See https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-put-follow.html - def follow(follower_index, body, params={}) + def follow(follower_index, body, params = {}) response = client.put "/#{follower_index}/_ccr/follow", params.merge(params, body:, action: "follow", rest_api: "ccr") response.body end @@ -54,10 +54,10 @@ def follow(follower_index, body, params={}) # follow_index_pattern: "{{leader_index}}-follower" }) # # See https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-put-auto-follow-pattern.html - def auto_follow(pattern_name, body, params={}) - response = client.put "/_ccr/auto_follow/#{pattern_name}", params.merge(params, body:, action: "create_auto_follow_pattern", rest_api: "ccr") - response.body - end + # def auto_follow(pattern_name, body, params={}) + # response = client.put "/_ccr/auto_follow/#{pattern_name}", params.merge(params, body:, action: "create_auto_follow_pattern", rest_api: "ccr") + # response.body + # end end end diff --git a/test/client/ccr_test.rb b/test/client/ccr_test.rb index e76b6de4..63e53cd6 100644 --- a/test/client/ccr_test.rb +++ b/test/client/ccr_test.rb @@ -2,7 +2,7 @@ require_relative "../test_helper" -describe ElastomerClient::Client::Ccr do +describe ElastomerClient::Client::Ccr do before do skip "Cannot test Ccr API without a replica cluster" unless $replica_client.available? @@ -36,19 +36,19 @@ assert_equal "Book 1", doc["_source"]["title"] end - it "successfully implements an auto-follow policy" do - ccr = $replica_client.ccr + # it "successfully implements an auto-follow policy" do + # ccr = $replica_client.ccr - ccr.auto_follow("follower_pattern", { remote_cluster: "leader", leader_index_patterns: ["*"], follow_index_pattern: "{{leader_index}}-follower" }) + # ccr.auto_follow("follower_pattern", { remote_cluster: "leader", leader_index_patterns: ["*"], follow_index_pattern: "{{leader_index}}-follower" }) - index = $client.index("followed_index") - index.create(default_index_settings) - wait_for_index(index.name, "green") - follower_index = $replica_client.index("followed_index-follower") - wait_for_index(follower_index.name, "green") - assert follower_index.exists? + # index = $client.index("followed_index") + # index.create(default_index_settings) + # wait_for_index(index.name, "green") + # follower_index = $replica_client.index("followed_index-follower") + # wait_for_index(follower_index.name, "green") + # assert follower_index.exists? - index.delete - follower_index.delete - end + # index.delete + # follower_index.delete + # end end From b1dae17e6f91044d18c6568e6ceea104f7686fb1 Mon Sep 17 00:00:00 2001 From: John Hernley Date: Mon, 10 Feb 2025 18:26:32 +0000 Subject: [PATCH 3/3] remove auto-follow --- lib/elastomer_client/client/ccr.rb | 25 +------------------------ test/client/ccr_test.rb | 15 --------------- 2 files changed, 1 insertion(+), 39 deletions(-) diff --git a/lib/elastomer_client/client/ccr.rb b/lib/elastomer_client/client/ccr.rb index 5e6ba238..f42a005c 100644 --- a/lib/elastomer_client/client/ccr.rb +++ b/lib/elastomer_client/client/ccr.rb @@ -33,32 +33,9 @@ def initialize(client) # # See https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-put-follow.html def follow(follower_index, body, params = {}) - response = client.put "/#{follower_index}/_ccr/follow", params.merge(params, body:, action: "follow", rest_api: "ccr") + response = client.put "/#{follower_index}/_ccr/follow", params.merge(body:, action: "follow", rest_api: "ccr") response.body end - - # Creates a new auto-follow pattern for the provided remote cluster. - # - # pattern_name - String name of the auto-follow pattern to create - # body - Hash of the request body - # :remote_cluster - String name of the remote cluster. Required. - # :leader_index_patterns - An array of simple index patterns to match against indices in the remote cluster - # :leader_index_exclusion_patterns - An array of simple index patterns that can be used to exclude indices from being auto-followed. - # :follow_index_pattern - The name of follower index. The template {{leader_index}} can be used to derive - # the name of the follower index from the name of the leader index. - # params - Hash of query parameters - # - # Examples - # - # ccr.auto_follow("follower_pattern", { remote_cluster: "leader", leader_index_patterns: ["leader_index*"] - # follow_index_pattern: "{{leader_index}}-follower" }) - # - # See https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-put-auto-follow-pattern.html - # def auto_follow(pattern_name, body, params={}) - # response = client.put "/_ccr/auto_follow/#{pattern_name}", params.merge(params, body:, action: "create_auto_follow_pattern", rest_api: "ccr") - # response.body - # end - end end end diff --git a/test/client/ccr_test.rb b/test/client/ccr_test.rb index 63e53cd6..dbee88f1 100644 --- a/test/client/ccr_test.rb +++ b/test/client/ccr_test.rb @@ -36,19 +36,4 @@ assert_equal "Book 1", doc["_source"]["title"] end - # it "successfully implements an auto-follow policy" do - # ccr = $replica_client.ccr - - # ccr.auto_follow("follower_pattern", { remote_cluster: "leader", leader_index_patterns: ["*"], follow_index_pattern: "{{leader_index}}-follower" }) - - # index = $client.index("followed_index") - # index.create(default_index_settings) - # wait_for_index(index.name, "green") - # follower_index = $replica_client.index("followed_index-follower") - # wait_for_index(follower_index.name, "green") - # assert follower_index.exists? - - # index.delete - # follower_index.delete - # end end