diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc index c8793e06f02a2..30a554ec0b60a 100644 --- a/docs/reference/search/profile.asciidoc +++ b/docs/reference/search/profile.asciidoc @@ -193,6 +193,17 @@ The API returns the following result: "debug": { "fast_path": 5 } + }, + { + "type": "StoredFieldsPhase", + "description": "", + "time_in_nanos": 5310, + "breakdown": { + "next_reader": 745, + "next_reader_count": 1, + "process": 4445, + "process_count": 5 + } } ] } @@ -1011,6 +1022,17 @@ And here is the fetch profile: "debug": { "fast_path": 4 } + }, + { + "type": "StoredFieldsPhase", + "description": "", + "time_in_nanos": 5310, + "breakdown": { + "next_reader": 745, + "next_reader_count": 1, + "process": 4445, + "process_count": 5 + } } ] } diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java index dd0739abbd938..d3f23d3f4a21c 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.xcontent.XContentType; import java.io.IOException; -import java.util.Collections; import java.util.function.Predicate; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; @@ -158,7 +157,7 @@ public void testSourceToXContent() throws IOException { } public void testSearchResponseToXContent() throws IOException { - SearchHit hit = new SearchHit(1, "id", Collections.emptyMap(), Collections.emptyMap()); + SearchHit hit = new SearchHit(1, "id"); hit.score(2.0f); SearchHit[] hits = new SearchHit[] { hit }; diff --git a/modules/parent-join/build.gradle b/modules/parent-join/build.gradle index 9704d35d80917..66079cfd444cb 100644 --- a/modules/parent-join/build.gradle +++ b/modules/parent-join/build.gradle @@ -18,4 +18,8 @@ restResources { restApi { include '_common', 'bulk', 'cluster', 'nodes', 'indices', 'index', 'search' } -} \ No newline at end of file +} + +tasks.named("yamlRestTestV7CompatTransform").configure { task -> + task.skipTest("/30_inner_hits/profile fetch", "profile output has changed") +} diff --git a/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/30_inner_hits.yml b/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/30_inner_hits.yml index 48cb6453b17bd..fab161baf1750 100644 --- a/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/30_inner_hits.yml +++ b/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/30_inner_hits.yml @@ -141,7 +141,7 @@ profile fetch: - gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 } - gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 } - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] } - - length: { profile.shards.0.fetch.children: 2 } + - length: { profile.shards.0.fetch.children: 3 } - match: { profile.shards.0.fetch.children.0.type: FetchSourcePhase } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 } @@ -152,3 +152,4 @@ profile fetch: - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 } - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 } - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 } + - match: { profile.shards.0.fetch.children.2.type: StoredFieldsPhase } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java index fd7a3b940bde9..e8d6edfd25a7b 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java @@ -18,6 +18,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.fetch.subphase.highlight.HighlightField; import org.elasticsearch.search.fetch.subphase.highlight.HighlightPhase; import org.elasticsearch.search.fetch.subphase.highlight.Highlighter; @@ -59,6 +60,11 @@ public void setNextReader(LeafReaderContext readerContext) { this.ctx = readerContext; } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + @Override public void process(HitContext hit) throws IOException { boolean singlePercolateQuery = percolateQueries.size() == 1; @@ -83,9 +89,10 @@ public void process(HitContext hit) throws IOException { int slot = (int) matchedSlot; BytesReference document = percolateQuery.getDocuments().get(slot); HitContext subContext = new HitContext( - new SearchHit(slot, "unknown", Collections.emptyMap(), Collections.emptyMap()), + new SearchHit(slot, "unknown"), percolatorLeafReaderContext, slot, + Map.of(), Source.fromBytes(document) ); // force source because MemoryIndex does not store fields diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java index a25d3c09e56bf..45c9b13fbb603 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java @@ -26,6 +26,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.ArrayList; @@ -69,6 +70,11 @@ public void setNextReader(LeafReaderContext readerContext) { this.ctx = readerContext; } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + @Override public void process(HitContext hitContext) throws IOException { for (PercolateContext pc : percolateContexts) { diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java index 8b4c1b5a6bddd..15b77ad304c54 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java @@ -34,6 +34,7 @@ import org.elasticsearch.test.ESTestCase; import java.util.Collections; +import java.util.Map; import java.util.stream.IntStream; import static org.mockito.Mockito.mock; @@ -55,7 +56,7 @@ public void testHitsExecute() throws Exception { LeafReaderContext context = reader.leaves().get(0); // A match: { - HitContext hit = new HitContext(new SearchHit(0), context, 0, Source.empty(null)); + HitContext hit = new HitContext(new SearchHit(0), context, 0, Map.of(), Source.empty(null)); PercolateQuery.QueryStore queryStore = ctx -> docId -> new TermQuery(new Term("field", "value")); MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "value", new WhitespaceAnalyzer()); @@ -86,7 +87,7 @@ public void testHitsExecute() throws Exception { // No match: { - HitContext hit = new HitContext(new SearchHit(0), context, 0, Source.empty(null)); + HitContext hit = new HitContext(new SearchHit(0), context, 0, Map.of(), Source.empty(null)); PercolateQuery.QueryStore queryStore = ctx -> docId -> new TermQuery(new Term("field", "value")); MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "value1", new WhitespaceAnalyzer()); @@ -116,7 +117,7 @@ public void testHitsExecute() throws Exception { // No query: { - HitContext hit = new HitContext(new SearchHit(0), context, 0, Source.empty(null)); + HitContext hit = new HitContext(new SearchHit(0), context, 0, Map.of(), Source.empty(null)); PercolateQuery.QueryStore queryStore = ctx -> docId -> null; MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "value", new WhitespaceAnalyzer()); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java index 7a8ebcd8d5ae7..d761b8e28aa95 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java @@ -60,7 +60,7 @@ public void testDCGAt() { SearchHit[] hits = new SearchHit[6]; for (int i = 0; i < 6; i++) { rated.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i])); - hits[i] = new SearchHit(i, Integer.toString(i), Collections.emptyMap(), Collections.emptyMap()); + hits[i] = new SearchHit(i, Integer.toString(i)); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); @@ -110,7 +110,7 @@ public void testDCGAtSixMissingRatings() { rated.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i])); } } - hits[i] = new SearchHit(i, Integer.toString(i), Collections.emptyMap(), Collections.emptyMap()); + hits[i] = new SearchHit(i, Integer.toString(i)); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); @@ -167,7 +167,7 @@ public void testDCGAtFourMoreRatings() { // only create four hits SearchHit[] hits = new SearchHit[4]; for (int i = 0; i < 4; i++) { - hits[i] = new SearchHit(i, Integer.toString(i), Collections.emptyMap(), Collections.emptyMap()); + hits[i] = new SearchHit(i, Integer.toString(i)); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java index 3cb873b4c22ff..79937c5f54604 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java @@ -103,7 +103,7 @@ private SearchHit[] createSearchHits(List rated, Integer[] releva if (relevanceRatings[i] != null) { rated.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i])); } - hits[i] = new SearchHit(i, Integer.toString(i), Collections.emptyMap(), Collections.emptyMap()); + hits[i] = new SearchHit(i, Integer.toString(i)); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); } return hits; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java index 04adb1f66b830..73a2eba86345a 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java @@ -190,7 +190,7 @@ public void testXContentParsingIsNotLenient() throws IOException { private static SearchHit[] createSearchHits(int from, int to, String index) { SearchHit[] hits = new SearchHit[to + 1 - from]; for (int i = from; i <= to; i++) { - hits[i] = new SearchHit(i, i + "", Collections.emptyMap(), Collections.emptyMap()); + hits[i] = new SearchHit(i, i + ""); hits[i].shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null)); } return hits; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java index 092aafbe2b484..306b6cafd4c9d 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java @@ -100,7 +100,7 @@ public void testIgnoreUnlabeled() { rated.add(createRatedDoc("test", "1", RELEVANT_RATING)); // add an unlabeled search hit SearchHit[] searchHits = Arrays.copyOf(toSearchHits(rated, "test"), 3); - searchHits[2] = new SearchHit(2, "2", Collections.emptyMap(), Collections.emptyMap()); + searchHits[2] = new SearchHit(2, "2"); searchHits[2].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", searchHits, rated); @@ -119,7 +119,7 @@ public void testIgnoreUnlabeled() { public void testNoRatedDocs() throws Exception { SearchHit[] hits = new SearchHit[5]; for (int i = 0; i < 5; i++) { - hits[i] = new SearchHit(i, i + "", Collections.emptyMap(), Collections.emptyMap()); + hits[i] = new SearchHit(i, i + ""); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); } EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", hits, Collections.emptyList()); @@ -248,7 +248,7 @@ private static PrecisionAtK mutate(PrecisionAtK original) { private static SearchHit[] toSearchHits(List rated, String index) { SearchHit[] hits = new SearchHit[rated.size()]; for (int i = 0; i < rated.size(); i++) { - hits[i] = new SearchHit(i, i + "", Collections.emptyMap(), Collections.emptyMap()); + hits[i] = new SearchHit(i, i + ""); hits[i].shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null)); } return hits; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java index e52f466b78e15..b602f2b0a119d 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java @@ -197,7 +197,7 @@ public void testToXContent() throws IOException { } private static RatedSearchHit searchHit(String index, int docId, Integer rating) { - SearchHit hit = new SearchHit(docId, docId + "", Collections.emptyMap(), Collections.emptyMap()); + SearchHit hit = new SearchHit(docId, docId + ""); hit.shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null)); hit.score(1.0f); return new RatedSearchHit(hit, rating != null ? OptionalInt.of(rating) : OptionalInt.empty()); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java index d441c96cb4328..d6cfb21049969 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java @@ -26,12 +26,7 @@ public class RatedSearchHitTests extends ESTestCase { public static RatedSearchHit randomRatedSearchHit() { OptionalInt rating = randomBoolean() ? OptionalInt.empty() : OptionalInt.of(randomIntBetween(0, 5)); - SearchHit searchHit = new SearchHit( - randomIntBetween(0, 10), - randomAlphaOfLength(10), - Collections.emptyMap(), - Collections.emptyMap() - ); + SearchHit searchHit = new SearchHit(randomIntBetween(0, 10), randomAlphaOfLength(10)); RatedSearchHit ratedSearchHit = new RatedSearchHit(searchHit, rating); return ratedSearchHit; } @@ -41,12 +36,7 @@ private static RatedSearchHit mutateTestItem(RatedSearchHit original) { SearchHit hit = original.getSearchHit(); switch (randomIntBetween(0, 1)) { case 0 -> rating = rating.isPresent() ? OptionalInt.of(rating.getAsInt() + 1) : OptionalInt.of(randomInt(5)); - case 1 -> hit = new SearchHit( - hit.docId(), - hit.getId() + randomAlphaOfLength(10), - Collections.emptyMap(), - Collections.emptyMap() - ); + case 1 -> hit = new SearchHit(hit.docId(), hit.getId() + randomAlphaOfLength(10)); default -> throw new IllegalStateException("The test should only allow two parameters mutated"); } return new RatedSearchHit(hit, rating); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RecallAtKTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RecallAtKTests.java index d021b0a6037e6..866675f4a9c11 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RecallAtKTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RecallAtKTests.java @@ -102,7 +102,7 @@ public void testNoRatedDocs() throws Exception { int k = 5; SearchHit[] hits = new SearchHit[k]; for (int i = 0; i < k; i++) { - hits[i] = new SearchHit(i, i + "", Collections.emptyMap(), Collections.emptyMap()); + hits[i] = new SearchHit(i, i + ""); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); } @@ -216,7 +216,7 @@ private static RecallAtK mutate(RecallAtK original) { private static SearchHit[] toSearchHits(List rated, String index) { SearchHit[] hits = new SearchHit[rated.size()]; for (int i = 0; i < rated.size(); i++) { - hits[i] = new SearchHit(i, i + "", Collections.emptyMap(), Collections.emptyMap()); + hits[i] = new SearchHit(i, i + ""); hits[i].shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null)); } return hits; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java index ff96bab7134c1..cbab4f892b215 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java @@ -565,7 +565,7 @@ protected RequestWrapper buildRequest(Hit doc) { action.start(); // create a simulated response. - SearchHit hit = new SearchHit(0, "id", emptyMap(), emptyMap()).sourceRef(new BytesArray("{}")); + SearchHit hit = new SearchHit(0, "id").sourceRef(new BytesArray("{}")); SearchHits hits = new SearchHits( IntStream.range(0, 100).mapToObj(i -> hit).toArray(SearchHit[]::new), new TotalHits(0, TotalHits.Relation.EQUAL_TO), diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java index da00f8b44036d..179d301f18dbc 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java @@ -46,7 +46,6 @@ import java.util.function.Function; import java.util.stream.IntStream; -import static java.util.Collections.emptyMap; import static org.apache.lucene.tests.util.TestUtil.randomSimpleString; import static org.elasticsearch.core.TimeValue.timeValueSeconds; import static org.hamcrest.Matchers.instanceOf; @@ -160,7 +159,7 @@ public void testScrollKeepAlive() { private SearchResponse createSearchResponse() { // create a simulated response. - SearchHit hit = new SearchHit(0, "id", emptyMap(), emptyMap()).sourceRef(new BytesArray("{}")); + SearchHit hit = new SearchHit(0, "id").sourceRef(new BytesArray("{}")); SearchHits hits = new SearchHits( IntStream.range(0, randomIntBetween(0, 20)).mapToObj(i -> hit).toArray(SearchHit[]::new), new TotalHits(0, TotalHits.Relation.EQUAL_TO), diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index ddf78b4301ae8..e92d6767a8032 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -75,6 +75,9 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task -> task.skipTest("search_shards/10_basic/Search shards aliases with and without filters", "Filter representation no longer outputs default boosts") task.skipTest("migration/10_get_feature_upgrade_status/Get feature upgrade status", "Awaits backport") task.skipTest("search/330_fetch_fields/Test disable source", "Error no longer thrown") + task.skipTest("search/370_profile/fetch fields", "profile output has changed") + task.skipTest("search/370_profile/fetch source", "profile output has changed") + task.skipTest("search/370_profile/fetch nested source", "profile output has changed") task.skipTest("search/240_date_nanos/doc value fields are working as expected across date and date_nanos fields", "Fetching docvalues field multiple times is no longer allowed") task.replaceValueInMatch("_type", "_doc") diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml index 9b9fe18276339..415da2ad8767b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml @@ -22,8 +22,8 @@ setup: --- fetch fields: - skip: - version: ' - 7.15.99' - reason: fetch profiling implemented in 7.16.0 + version: ' - 8.5.99' + reason: stored fields phase added in 8.6 - do: search: @@ -42,8 +42,9 @@ fetch fields: - gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 } - gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 } - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] } - - length: { profile.shards.0.fetch.children: 1 } + - length: { profile.shards.0.fetch.children: 2 } - match: { profile.shards.0.fetch.children.0.type: FetchFieldsPhase } + - match: { profile.shards.0.fetch.children.1.type: StoredFieldsPhase } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 } @@ -52,8 +53,8 @@ fetch fields: --- fetch source: - skip: - version: ' - 7.99.99' - reason: fetch profiling implemented in 8.0.0 to be backported to 7.16.0 + version: ' - 8.5.99' + reason: stored fields phase added in 8.6 - do: search: @@ -70,19 +71,20 @@ fetch source: - gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 } - gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 } - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] } - - length: { profile.shards.0.fetch.children: 1 } + - length: { profile.shards.0.fetch.children: 2 } - match: { profile.shards.0.fetch.children.0.type: FetchSourcePhase } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 } - match: { profile.shards.0.fetch.children.0.debug.fast_path: 1 } + - match: { profile.shards.0.fetch.children.1.type: StoredFieldsPhase } --- fetch nested source: - skip: - version: ' - 7.15.99' - reason: fetch profiling implemented in 7.16.0 + version: ' - 8.5.99' + reason: stored fields phase added in 8.6 - do: indices.create: @@ -133,7 +135,7 @@ fetch nested source: - gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 } - gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 } - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] } - - length: { profile.shards.0.fetch.children: 2 } + - length: { profile.shards.0.fetch.children: 3 } - match: { profile.shards.0.fetch.children.0.type: FetchSourcePhase } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 } @@ -144,6 +146,7 @@ fetch nested source: - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 } - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 } - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 } + - match: { profile.shards.0.fetch.children.2.type: StoredFieldsPhase } --- disabling stored fields removes fetch sub phases: diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java index 561d61185c9d5..2d772e25131c0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java @@ -56,6 +56,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xcontent.ObjectParser; @@ -92,6 +93,11 @@ public List getFetchSubPhases(FetchPhaseConstructionContext conte @Override public void setNextReader(LeafReaderContext readerContext) {} + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + @Override public void process(FetchSubPhase.HitContext hitContext) { if (fetchContext.getIndexName().startsWith("boom")) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java index 4b19abf28a5f2..e2c295985c3bc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java @@ -117,6 +117,11 @@ public void setNextReader(LeafReaderContext readerContext) { } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + @Override public void process(HitContext hitContext) throws IOException { hitExecute(searchContext, hitContext); diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/CustomFieldsVisitor.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/CustomFieldsVisitor.java index 6e2835488bf10..d7f6e3541838b 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/CustomFieldsVisitor.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/CustomFieldsVisitor.java @@ -10,13 +10,12 @@ import org.apache.lucene.index.FieldInfo; import java.util.HashSet; +import java.util.List; import java.util.Set; /** - * A field visitor that allows to load a selection of the stored fields by exact name or by pattern. - * Supported pattern styles: "xxx*", "*xxx", "*xxx*" and "xxx*yyy". - * The Uid field is always loaded. - * The class is optimized for source loading as it is a common use case. + * A field visitor that allows to load a selection of the stored fields by exact name + * {@code _id} and {@code _routing} fields are always loaded. */ public class CustomFieldsVisitor extends FieldsVisitor { @@ -24,7 +23,11 @@ public class CustomFieldsVisitor extends FieldsVisitor { public CustomFieldsVisitor(Set fields, boolean loadSource) { super(loadSource); - this.fields = fields; + this.fields = new HashSet<>(fields); + // metadata fields are already handled by FieldsVisitor, so removing + // them here means that if the only fields requested are metadata + // fields then we can shortcut loading + List.of("_id", "_routing", "_source").forEach(this.fields::remove); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 71177cfdff77e..b16715b8769d7 100644 --- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; @@ -29,7 +30,6 @@ import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.search.fetch.FetchPhase; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.lookup.Source; @@ -267,7 +267,11 @@ private GetResult innerGetFetch( if (false == needed.contains(entry.getKey())) { continue; } - List values = FetchPhase.processStoredField(mapperService::fieldType, entry.getKey(), entry.getValue()); + MappedFieldType ft = mapperService.fieldType(entry.getKey()); + if (ft == null) { + continue; // user asked for a non-existent field, ignore it + } + List values = entry.getValue().stream().map(ft::valueForDisplay).toList(); if (mapperService.isMetadataField(entry.getKey())) { metadataFields.put(entry.getKey(), new DocumentField(entry.getKey(), values)); } else { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java index d65c5897e54fd..ecbeff9d8cb41 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java @@ -40,14 +40,14 @@ public static class Defaults { } } + public static final IgnoredFieldType FIELD_TYPE = new IgnoredFieldType(); + private static final IgnoredFieldMapper INSTANCE = new IgnoredFieldMapper(); public static final TypeParser PARSER = new FixedTypeParser(c -> INSTANCE); public static final class IgnoredFieldType extends StringFieldType { - public static final IgnoredFieldType INSTANCE = new IgnoredFieldType(); - private IgnoredFieldType() { super(NAME, true, true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); } @@ -73,7 +73,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) } private IgnoredFieldMapper() { - super(IgnoredFieldType.INSTANCE); + super(FIELD_TYPE); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java index ac800e147d44e..a6075b9a46627 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java @@ -29,6 +29,8 @@ public class LegacyTypeFieldMapper extends MetadataFieldMapper { public static final String CONTENT_TYPE = "_type"; + public static final MappedFieldType FIELD_TYPE = new LegacyTypeFieldType(); + private static final LegacyTypeFieldMapper INSTANCE = new LegacyTypeFieldMapper(); public static final TypeParser PARSER = new FixedTypeParser(c -> INSTANCE); @@ -36,7 +38,7 @@ public class LegacyTypeFieldMapper extends MetadataFieldMapper { private static final Map ANALYZERS = Map.of(NAME, Lucene.KEYWORD_ANALYZER); protected LegacyTypeFieldMapper() { - super(new LegacyTypeFieldType()); + super(FIELD_TYPE); } @Override @@ -44,7 +46,7 @@ public Map indexAnalyzers() { return ANALYZERS; } - static final class LegacyTypeFieldType extends TermBasedFieldType { + private static final class LegacyTypeFieldType extends TermBasedFieldType { LegacyTypeFieldType() { super(NAME, false, true, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java index 4a38289ec3202..f7ef8b9113cbc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java @@ -67,9 +67,9 @@ public RoutingFieldMapper build() { public static final TypeParser PARSER = new ConfigurableTypeParser(c -> RoutingFieldMapper.get(Defaults.REQUIRED), c -> new Builder()); - static final class RoutingFieldType extends StringFieldType { + public static final MappedFieldType FIELD_TYPE = new RoutingFieldType(); - static RoutingFieldType INSTANCE = new RoutingFieldType(); + static final class RoutingFieldType extends StringFieldType { private RoutingFieldType() { super(NAME, true, true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); @@ -101,7 +101,7 @@ public static RoutingFieldMapper get(boolean required) { } private RoutingFieldMapper(boolean required) { - super(RoutingFieldType.INSTANCE); + super(FIELD_TYPE); this.required = required; } diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 5ec5a38978c73..e8b23a522b050 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -404,7 +404,7 @@ public void addRescore(RescoreContext rescore) { @Override public boolean hasScriptFields() { - return scriptFields != null; + return scriptFields != null && scriptFields.fields().isEmpty() == false; } @Override diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java index b5f3164987e86..3ef0e0e6d95d5 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHit.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java @@ -85,8 +85,8 @@ public final class SearchHit implements Writeable, ToXContentObject, Iterable documentFields; - private final Map metaFields; + private final Map documentFields = new HashMap<>(); + private final Map metaFields = new HashMap<>(); private Map highlightFields = null; @@ -111,20 +111,14 @@ public final class SearchHit implements Writeable, ToXContentObject, Iterable documentFields, Map metaFields) { - this(docId, id, null, documentFields, metaFields); + public SearchHit(int docId, String id) { + this(docId, id, null); } - public SearchHit( - int nestedTopDocId, - String id, - NestedIdentity nestedIdentity, - Map documentFields, - Map metaFields - ) { + public SearchHit(int nestedTopDocId, String id, NestedIdentity nestedIdentity) { this.docId = nestedTopDocId; if (id != null) { this.id = new Text(id); @@ -132,8 +126,6 @@ public SearchHit( this.id = null; } this.nestedIdentity = nestedIdentity; - this.documentFields = documentFields == null ? emptyMap() : documentFields; - this.metaFields = metaFields == null ? emptyMap() : metaFields; } public SearchHit(StreamInput in) throws IOException { @@ -155,12 +147,10 @@ public SearchHit(StreamInput in) throws IOException { explanation = readExplanation(in); } if (in.getVersion().onOrAfter(Version.V_7_8_0)) { - documentFields = in.readMap(StreamInput::readString, DocumentField::new); - metaFields = in.readMap(StreamInput::readString, DocumentField::new); + documentFields.putAll(in.readMap(StreamInput::readString, DocumentField::new)); + metaFields.putAll(in.readMap(StreamInput::readString, DocumentField::new)); } else { Map fields = readFields(in); - documentFields = new HashMap<>(); - metaFields = new HashMap<>(); fields.forEach( (fieldName, docField) -> (MapperService.isMetadataFieldStatic(fieldName) ? metaFields : documentFields).put( fieldName, @@ -437,10 +427,14 @@ public DocumentField field(String fieldName) { * */ public void setDocumentField(String fieldName, DocumentField field) { if (fieldName == null || field == null) return; - if (documentFields.size() == 0) this.documentFields = new HashMap<>(); this.documentFields.put(fieldName, field); } + public void addDocumentFields(Map docFields, Map metaFields) { + this.documentFields.putAll(docFields); + this.metaFields.putAll(metaFields); + } + /** * @return a map of metadata fields for this hit */ @@ -846,7 +840,8 @@ public static SearchHit createFromMap(Map values) { Map metaFields = get(METADATA_FIELDS, values, Collections.emptyMap()); Map documentFields = get(DOCUMENT_FIELDS, values, Collections.emptyMap()); - SearchHit searchHit = new SearchHit(-1, id, nestedIdentity, documentFields, metaFields); + SearchHit searchHit = new SearchHit(-1, id, nestedIdentity); + searchHit.addDocumentFields(documentFields, metaFields); String index = get(Fields._INDEX, values, null); String clusterAlias = null; if (index != null) { diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index 53a456c55b07b..8d6b7f17020cc 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -216,6 +216,7 @@ import org.elasticsearch.search.fetch.subphase.MatchedQueriesPhase; import org.elasticsearch.search.fetch.subphase.ScriptFieldsPhase; import org.elasticsearch.search.fetch.subphase.SeqNoPrimaryTermPhase; +import org.elasticsearch.search.fetch.subphase.StoredFieldsPhase; import org.elasticsearch.search.fetch.subphase.highlight.FastVectorHighlighter; import org.elasticsearch.search.fetch.subphase.highlight.HighlightPhase; import org.elasticsearch.search.fetch.subphase.highlight.Highlighter; @@ -989,6 +990,7 @@ private void registerSignificanceHeuristic(Sig private void registerFetchSubPhases(List plugins) { registerFetchSubPhase(new ExplainPhase()); + registerFetchSubPhase(new StoredFieldsPhase()); registerFetchSubPhase(new FetchDocValuesPhase()); registerFetchSubPhase(new ScriptFieldsPhase()); registerFetchSubPhase(new FetchSourcePhase()); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java index 0786f2952074e..3655bd58408e8 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java @@ -9,6 +9,7 @@ package org.elasticsearch.search.fetch; import org.apache.lucene.search.Query; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.SearchExecutionContext; @@ -36,16 +37,53 @@ public class FetchContext { private final SearchContext searchContext; - private final SearchLookup searchLookup; private final SourceLoader sourceLoader; + private final FetchSourceContext fetchSourceContext; + private final StoredFieldsContext storedFieldsContext; /** * Create a FetchContext based on a SearchContext */ public FetchContext(SearchContext searchContext) { this.searchContext = searchContext; - this.searchLookup = searchContext.getSearchExecutionContext().lookup(); this.sourceLoader = searchContext.newSourceLoader(); + this.storedFieldsContext = buildStoredFieldsContext(searchContext); + this.fetchSourceContext = buildFetchSourceContext(searchContext); + } + + private static FetchSourceContext buildFetchSourceContext(SearchContext in) { + FetchSourceContext fsc = in.fetchSourceContext(); + StoredFieldsContext sfc = in.storedFieldsContext(); + if (fsc == null) { + boolean hasStoredFields = in.hasStoredFields(); + boolean hasScriptFields = in.hasScriptFields(); + // TODO it seems a bit odd that we disable implicit source loading if we've asked + // for stored fields or script fields? But not eg doc_value fields or via + // the `fields` API + if (hasStoredFields == false && hasScriptFields == false) { + fsc = FetchSourceContext.of(true); + } + } + if (sfc != null && sfc.fetchFields()) { + for (String field : sfc.fieldNames()) { + if (SourceFieldMapper.NAME.equals(field)) { + fsc = fsc == null ? FetchSourceContext.of(true) : FetchSourceContext.of(true, fsc.includes(), fsc.excludes()); + } + } + } + if (sfc != null && sfc.fetchFields() == false) { + fsc = null; + } + return fsc; + } + + private static StoredFieldsContext buildStoredFieldsContext(SearchContext in) { + StoredFieldsContext sfc = in.storedFieldsContext(); + if (sfc == null) { + // if nothing is requested then we just do a standard metadata stored fields request + sfc = StoredFieldsContext.metadataOnly(); + } + return sfc; } /** @@ -66,7 +104,7 @@ public ContextIndexSearcher searcher() { * The {@code SearchLookup} for the this context */ public SearchLookup searchLookup() { - return searchLookup; + return searchContext.getSearchExecutionContext().lookup(); } /** @@ -101,7 +139,14 @@ public ParsedQuery parsedPostFilter() { * Configuration for fetching _source */ public FetchSourceContext fetchSourceContext() { - return searchContext.fetchSourceContext(); + return this.fetchSourceContext; + } + + /** + * Configuration for fetching stored fields + */ + public StoredFieldsContext storedFieldsContext() { + return storedFieldsContext; } /** diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index 5f6ea8b32f0c5..5540a11eb2c1e 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -12,16 +12,12 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.TotalHits; -import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.SourceLoader; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.LeafNestedDocuments; import org.elasticsearch.search.NestedDocuments; import org.elasticsearch.search.SearchContextSourcePrinter; @@ -29,7 +25,6 @@ import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.fetch.FetchSubPhase.HitContext; -import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.InnerHitsContext; import org.elasticsearch.search.fetch.subphase.InnerHitsPhase; import org.elasticsearch.search.internal.SearchContext; @@ -43,18 +38,11 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.function.Function; import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static java.util.Collections.emptyMap; /** * Fetch phase of a search request, used to fetch the actual top matching documents to be returned to the client, identified @@ -102,15 +90,20 @@ public void execute(SearchContext context) { private SearchHits buildSearchHits(SearchContext context, Profiler profiler) { - SourceLoader sourceLoader = context.newSourceLoader(); - Map> storedToRequestedFields = new HashMap<>(); - StoredFieldLoader storedFieldLoader = profiler.storedFields( - createStoredFieldLoader(context, sourceLoader, storedToRequestedFields) - ); - FetchContext fetchContext = new FetchContext(context); + SourceLoader sourceLoader = context.newSourceLoader(); List processors = getProcessors(context.shardTarget(), fetchContext, profiler); + + StoredFieldsSpec storedFieldsSpec = StoredFieldsSpec.NO_REQUIREMENTS; + for (FetchSubPhaseProcessor proc : processors) { + storedFieldsSpec = storedFieldsSpec.merge(proc.storedFieldsSpec()); + } + storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(false, false, sourceLoader.requiredStoredFields())); + + StoredFieldLoader storedFieldLoader = profiler.storedFields(buildStoredFieldsLoader(storedFieldsSpec)); + boolean requiresSource = storedFieldsSpec.requiresSource(); + NestedDocuments nestedDocuments = context.getSearchExecutionContext().getNestedDocuments(); FetchPhaseDocsIterator docsIterator = new FetchPhaseDocsIterator() { @@ -126,7 +119,7 @@ protected void setNextReader(LeafReaderContext ctx, int[] docsInLeaf) throws IOE this.ctx = ctx; this.leafNestedDocuments = nestedDocuments.getLeafNestedDocuments(ctx); this.leafStoredFieldLoader = storedFieldLoader.getLoader(ctx, docsInLeaf); - this.leafSourceLoader = fetchContext.sourceLoader().leaf(ctx.reader(), docsInLeaf); + this.leafSourceLoader = sourceLoader.leaf(ctx.reader(), docsInLeaf); for (FetchSubPhaseProcessor processor : processors) { processor.setNextReader(ctx); } @@ -140,11 +133,11 @@ protected SearchHit nextDoc(int doc) throws IOException { } HitContext hit = prepareHitContext( context, + requiresSource, profiler, leafNestedDocuments, leafStoredFieldLoader, doc, - storedToRequestedFields, ctx, leafSourceLoader ); @@ -165,6 +158,13 @@ protected SearchHit nextDoc(int doc) throws IOException { return new SearchHits(hits, totalHits, context.queryResult().getMaxScore()); } + private static StoredFieldLoader buildStoredFieldsLoader(StoredFieldsSpec spec) { + if (spec.noRequirements()) { + return StoredFieldLoader.empty(); + } + return StoredFieldLoader.create(spec.requiresSource(), spec.requiredStoredFields()); + } + List getProcessors(SearchShardTarget target, FetchContext context, Profiler profiler) { try { List processors = new ArrayList<>(); @@ -180,91 +180,33 @@ List getProcessors(SearchShardTarget target, FetchContex } } - private static StoredFieldLoader createStoredFieldLoader( - SearchContext context, - SourceLoader sourceLoader, - Map> storedToRequestedFields - ) { - StoredFieldsContext storedFieldsContext = context.storedFieldsContext(); - - if (storedFieldsContext == null) { - // no fields specified, default to return source if no explicit indication - if (context.hasScriptFields() == false && context.hasFetchSourceContext() == false) { - context.fetchSourceContext(FetchSourceContext.FETCH_SOURCE); - } - boolean loadSource = sourceRequired(context); - if (loadSource) { - if (false == sourceLoader.requiredStoredFields().isEmpty()) { - // add the stored fields needed to load the source mapping to an empty set so they aren't returned - sourceLoader.requiredStoredFields().forEach(fieldName -> storedToRequestedFields.putIfAbsent(fieldName, Set.of())); - } - } - return StoredFieldLoader.create(loadSource, sourceLoader.requiredStoredFields()); - } else if (storedFieldsContext.fetchFields() == false) { - // disable stored fields entirely - return StoredFieldLoader.empty(); - } else { - for (String fieldNameOrPattern : context.storedFieldsContext().fieldNames()) { - if (fieldNameOrPattern.equals(SourceFieldMapper.NAME)) { - FetchSourceContext fetchSourceContext = context.hasFetchSourceContext() - ? context.fetchSourceContext() - : FetchSourceContext.FETCH_SOURCE; - context.fetchSourceContext(FetchSourceContext.of(true, fetchSourceContext.includes(), fetchSourceContext.excludes())); - continue; - } - SearchExecutionContext searchExecutionContext = context.getSearchExecutionContext(); - Collection fieldNames = searchExecutionContext.getMatchingFieldNames(fieldNameOrPattern); - for (String fieldName : fieldNames) { - MappedFieldType fieldType = searchExecutionContext.getFieldType(fieldName); - String storedField = fieldType.name(); - Set requestedFields = storedToRequestedFields.computeIfAbsent(storedField, key -> new HashSet<>()); - requestedFields.add(fieldName); - } - } - boolean loadSource = sourceRequired(context); - if (loadSource) { - sourceLoader.requiredStoredFields().forEach(fieldName -> storedToRequestedFields.putIfAbsent(fieldName, Set.of())); - } - if (storedToRequestedFields.isEmpty()) { - // empty list specified, default to disable _source if no explicit indication - return StoredFieldLoader.create(loadSource, sourceLoader.requiredStoredFields()); - } else { - return StoredFieldLoader.create(loadSource, storedToRequestedFields.keySet()); - } - } - } - - private static boolean sourceRequired(SearchContext context) { - return context.sourceRequested() || context.fetchFieldsContext() != null; - } - private static HitContext prepareHitContext( SearchContext context, + boolean requiresSource, Profiler profiler, LeafNestedDocuments nestedDocuments, LeafStoredFieldLoader leafStoredFieldLoader, int docId, - Map> storedToRequestedFields, LeafReaderContext subReaderContext, SourceLoader.Leaf sourceLoader ) throws IOException { if (nestedDocuments.advance(docId - subReaderContext.docBase) == null) { return prepareNonNestedHitContext( context, + requiresSource, profiler, leafStoredFieldLoader, docId, - storedToRequestedFields, subReaderContext, sourceLoader ); } else { return prepareNestedHitContext( context, + requiresSource, profiler, docId, nestedDocuments, - storedToRequestedFields, subReaderContext, leafStoredFieldLoader ); @@ -280,10 +222,10 @@ private static HitContext prepareHitContext( */ private static HitContext prepareNonNestedHitContext( SearchContext context, + boolean requiresSource, Profiler profiler, LeafStoredFieldLoader leafStoredFieldLoader, int docId, - Map> storedToRequestedFields, LeafReaderContext subReaderContext, SourceLoader.Leaf sourceLoader ) throws IOException { @@ -292,22 +234,13 @@ private static HitContext prepareNonNestedHitContext( leafStoredFieldLoader.advanceTo(subDocId); if (leafStoredFieldLoader.id() == null) { - SearchHit hit = new SearchHit(docId, null, null, null); + SearchHit hit = new SearchHit(docId, null); Source source = Source.lazy(lazyStoredSourceLoader(profiler, subReaderContext, subDocId)); - return new HitContext(hit, subReaderContext, subDocId, source); + return new HitContext(hit, subReaderContext, subDocId, Map.of(), source); } else { - SearchHit hit; - if (leafStoredFieldLoader.storedFields().isEmpty() == false) { - Map docFields = new HashMap<>(); - Map metaFields = new HashMap<>(); - fillDocAndMetaFields(context, leafStoredFieldLoader.storedFields(), storedToRequestedFields, docFields, metaFields); - hit = new SearchHit(docId, leafStoredFieldLoader.id(), docFields, metaFields); - } else { - hit = new SearchHit(docId, leafStoredFieldLoader.id(), emptyMap(), emptyMap()); - } - + SearchHit hit = new SearchHit(docId, leafStoredFieldLoader.id()); Source source; - if (sourceRequired(context)) { + if (requiresSource) { try { profiler.startLoadingSource(); source = Source.fromBytes(sourceLoader.source(leafStoredFieldLoader, subDocId)); @@ -320,7 +253,7 @@ private static HitContext prepareNonNestedHitContext( } else { source = Source.lazy(lazyStoredSourceLoader(profiler, subReaderContext, subDocId)); } - return new HitContext(hit, subReaderContext, subDocId, source); + return new HitContext(hit, subReaderContext, subDocId, leafStoredFieldLoader.storedFields(), source); } } @@ -348,17 +281,13 @@ private static Supplier lazyStoredSourceLoader(Profiler profiler, LeafRe @SuppressWarnings("unchecked") private static HitContext prepareNestedHitContext( SearchContext context, + boolean requiresSource, Profiler profiler, int topDocId, LeafNestedDocuments nestedInfo, - Map> storedToRequestedFields, LeafReaderContext subReaderContext, LeafStoredFieldLoader childFieldLoader ) throws IOException { - // Also if highlighting is requested on nested documents we need to fetch the _source from the root document, - // otherwise highlighting will attempt to fetch the _source from the nested doc, which will fail, - // because the entire _source is only stored with the root document. - boolean needSource = sourceRequired(context) || context.highlight() != null; String rootId; Map rootSourceAsMap = null; @@ -367,18 +296,18 @@ private static HitContext prepareNestedHitContext( if (context instanceof InnerHitsContext.InnerHitSubContext innerHitsContext) { rootId = innerHitsContext.getRootId(); - if (needSource) { + if (requiresSource) { Source rootLookup = innerHitsContext.getRootLookup(); rootSourceAsMap = rootLookup.source(); rootSourceContentType = rootLookup.sourceContentType(); } } else { - StoredFieldLoader rootLoader = profiler.storedFields(StoredFieldLoader.create(needSource, Collections.emptySet())); + StoredFieldLoader rootLoader = profiler.storedFields(StoredFieldLoader.create(requiresSource, Collections.emptySet())); LeafStoredFieldLoader leafRootLoader = rootLoader.getLoader(subReaderContext, null); leafRootLoader.advanceTo(nestedInfo.rootDoc()); rootId = leafRootLoader.id(); - if (needSource) { + if (requiresSource) { if (leafRootLoader.source() != null) { Tuple> tuple = XContentHelper.convertToMap(leafRootLoader.source(), false); rootSourceAsMap = tuple.v2(); @@ -389,20 +318,11 @@ private static HitContext prepareNestedHitContext( } } - Map docFields = emptyMap(); - Map metaFields = emptyMap(); - if (context.hasStoredFields() && context.storedFieldsContext().fieldNames().isEmpty() == false) { - childFieldLoader.advanceTo(nestedInfo.doc()); - if (childFieldLoader.storedFields().isEmpty() == false) { - docFields = new HashMap<>(); - metaFields = new HashMap<>(); - fillDocAndMetaFields(context, childFieldLoader.storedFields(), storedToRequestedFields, docFields, metaFields); - } - } + childFieldLoader.advanceTo(nestedInfo.doc()); SearchHit.NestedIdentity nestedIdentity = nestedInfo.nestedIdentity(); - SearchHit hit = new SearchHit(topDocId, rootId, nestedIdentity, docFields, metaFields); + SearchHit hit = new SearchHit(topDocId, rootId, nestedIdentity); if (rootSourceAsMap != null && rootSourceAsMap.isEmpty() == false) { // Isolate the nested json array object that matches with nested hit and wrap it back into the same json @@ -427,47 +347,15 @@ private static HitContext prepareNestedHitContext( current = next; } } - return new HitContext(hit, subReaderContext, nestedInfo.doc(), Source.fromMap(nestedSourceAsMap, rootSourceContentType)); - } - return new HitContext(hit, subReaderContext, nestedInfo.doc(), Source.empty(rootSourceContentType)); - } - - public static List processStoredField(Function fieldTypeLookup, String field, List input) { - MappedFieldType ft = fieldTypeLookup.apply(field); - if (ft == null) { - // TODO remove this once we've stopped calling it for accidentally loaded fields - return input; - } - return input.stream().map(ft::valueForDisplay).collect(Collectors.toList()); - } - - private static void fillDocAndMetaFields( - SearchContext context, - Map> storedFields, - Map> storedToRequestedFields, - Map docFields, - Map metaFields - ) { - Function fieldTypeLookup = context.getSearchExecutionContext()::getFieldType; - for (Map.Entry> entry : storedFields.entrySet()) { - String storedField = entry.getKey(); - List storedValues = processStoredField(fieldTypeLookup, storedField, entry.getValue()); - if (storedToRequestedFields.containsKey(storedField)) { - for (String requestedField : storedToRequestedFields.get(storedField)) { - if (context.getSearchExecutionContext().isMetadataField(requestedField)) { - metaFields.put(requestedField, new DocumentField(requestedField, storedValues)); - } else { - docFields.put(requestedField, new DocumentField(requestedField, storedValues)); - } - } - } else { - if (context.getSearchExecutionContext().isMetadataField(storedField)) { - metaFields.put(storedField, new DocumentField(storedField, storedValues)); - } else { - docFields.put(storedField, new DocumentField(storedField, storedValues)); - } - } + return new HitContext( + hit, + subReaderContext, + nestedInfo.doc(), + childFieldLoader.storedFields(), + Source.fromMap(nestedSourceAsMap, rootSourceContentType) + ); } + return new HitContext(hit, subReaderContext, nestedInfo.doc(), childFieldLoader.storedFields(), Source.empty(null)); } interface Profiler { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchProfiler.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchProfiler.java index 728b1ba6a2372..3e3d654c0a393 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchProfiler.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchProfiler.java @@ -122,6 +122,11 @@ public void setNextReader(LeafReaderContext readerContext) throws IOException { } } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return delegate.storedFieldsSpec(); + } + @Override public void process(HitContext hitContext) throws IOException { Timer timer = breakdown.getTimer(FetchSubPhaseTiming.PROCESS); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java index 4d22a601b980f..a6f41f8b7fed3 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java @@ -15,6 +15,8 @@ import org.elasticsearch.search.lookup.Source; import java.io.IOException; +import java.util.List; +import java.util.Map; /** * Sub phase within the fetch phase used to fetch things *about* the documents like highlighting or matched queries. @@ -26,12 +28,14 @@ class HitContext { private final LeafReaderContext readerContext; private final int docId; private final Source source; + private final Map> loadedFields; - public HitContext(SearchHit hit, LeafReaderContext context, int docId, Source source) { + public HitContext(SearchHit hit, LeafReaderContext context, int docId, Map> loadedFields, Source source) { this.hit = hit; this.readerContext = context; this.docId = docId; this.source = source; + this.loadedFields = loadedFields; } public SearchHit hit() { @@ -64,6 +68,10 @@ public Source source() { return source; } + public Map> loadedFields() { + return loadedFields; + } + public IndexReader topLevelReader() { return ReaderUtil.getTopLevelContext(readerContext).reader(); } @@ -76,4 +84,5 @@ public IndexReader topLevelReader() { * implementation should return {@code null} */ FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) throws IOException; + } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseProcessor.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseProcessor.java index b2a1058e62c35..316c61e4db1dc 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseProcessor.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseProcessor.java @@ -36,4 +36,9 @@ public interface FetchSubPhaseProcessor { default Map getDebugInfo() { return null; } + + /** + * The stored fields or source required by this sub phase + */ + StoredFieldsSpec storedFieldsSpec(); } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsContext.java b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsContext.java index a04904d24ccb3..ae0e52ab69091 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsContext.java @@ -31,7 +31,7 @@ public class StoredFieldsContext implements Writeable { public static final String _NONE_ = "_none_"; private final List fieldNames; - private boolean fetchFields; + private final boolean fetchFields; private StoredFieldsContext(boolean fetchFields) { this.fetchFields = fetchFields; @@ -143,6 +143,10 @@ public void toXContent(String preferredName, XContentBuilder builder) throws IOE } } + public static StoredFieldsContext metadataOnly() { + return new StoredFieldsContext(true); + } + public static StoredFieldsContext fromList(List fieldNames) { if (fieldNames.size() == 1 && _NONE_.equals(fieldNames.get(0))) { return new StoredFieldsContext(false); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java new file mode 100644 index 0000000000000..0b68de0fc0242 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.fetch; + +import java.util.HashSet; +import java.util.Set; + +/** + * Defines which stored fields need to be loaded during a fetch + * @param requiresSource should source be loaded + * @param requiredStoredFields a set of stored fields to load + */ +public record StoredFieldsSpec(boolean requiresSource, boolean requiresMetadata, Set requiredStoredFields) { + + public boolean noRequirements() { + return requiresSource == false && requiresMetadata == false && requiredStoredFields.isEmpty(); + } + + /** + * Use when no stored fields are required + */ + public static StoredFieldsSpec NO_REQUIREMENTS = new StoredFieldsSpec(false, false, Set.of()); + + /** + * Use when the source should be loaded but no other stored fields are required + */ + public static StoredFieldsSpec NEEDS_SOURCE = new StoredFieldsSpec(true, false, Set.of()); + + /** + * Combine these stored field requirements with those from another StoredFieldsSpec + */ + public StoredFieldsSpec merge(StoredFieldsSpec other) { + Set mergedFields = new HashSet<>(this.requiredStoredFields); + mergedFields.addAll(other.requiredStoredFields); + return new StoredFieldsSpec( + this.requiresSource || other.requiresSource, + this.requiresMetadata || other.requiresMetadata, + mergedFields + ); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java index a28b4fef89c88..0873ca777d428 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java @@ -12,6 +12,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.rescore.RescoreContext; import java.io.IOException; @@ -42,6 +43,11 @@ public void process(HitContext hitContext) throws IOException { // we use the top level doc id, since we work with the top level searcher hitContext.hit().explanation(explanation); } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } }; } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java index 3a5df16856312..30177c780f4a3 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java @@ -15,6 +15,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.ArrayList; @@ -59,6 +60,11 @@ public void setNextReader(LeafReaderContext readerContext) { } } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + @Override public void process(HitContext hit) throws IOException { for (DocValueField f : fields) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java index 501aa0b1979dc..c0bfad44195cd 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.Map; @@ -39,6 +40,12 @@ public void setNextReader(LeafReaderContext readerContext) { fieldFetcher.setNextReader(readerContext); } + @Override + public StoredFieldsSpec storedFieldsSpec() { + // TODO can we get finer-grained information from the FieldFetcher for this? + return StoredFieldsSpec.NEEDS_SOURCE; + } + @Override public void process(HitContext hitContext) throws IOException { Map documentFields = fieldFetcher.fetch(hitContext.source(), hitContext.docId()); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchScorePhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchScorePhase.java index 906ae69552c5f..897c94da3a9b3 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchScorePhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchScorePhase.java @@ -17,6 +17,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; @@ -41,6 +42,11 @@ public void setNextReader(LeafReaderContext readerContext) throws IOException { scorer = scorerSupplier.get(1L); // random-access } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + @Override public void process(HitContext hitContext) throws IOException { if (scorer == null || scorer.iterator().advance(hitContext.docId()) != hitContext.docId()) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java index 575555f6f4dcd..3b8e4e69d9318 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java @@ -13,6 +13,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.search.lookup.SourceFilter; @@ -25,7 +26,6 @@ public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) { if (fetchSourceContext == null || fetchSourceContext.fetchSource() == false) { return null; } - String index = fetchContext.getIndexName(); assert fetchSourceContext.fetchSource(); SourceFilter sourceFilter = fetchSourceContext.filter(); @@ -37,8 +37,14 @@ public void setNextReader(LeafReaderContext readerContext) { } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NEEDS_SOURCE; + } + @Override public void process(HitContext hitContext) { + String index = fetchContext.getIndexName(); if (fetchContext.getSearchExecutionContext().isSourceEnabled() == false) { if (fetchSourceContext.hasFilter()) { throw new IllegalArgumentException( diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchVersionPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchVersionPhase.java index 8b89aecc4761e..67cf97894e079 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchVersionPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchVersionPhase.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; @@ -32,6 +33,11 @@ public void setNextReader(LeafReaderContext readerContext) throws IOException { versions = readerContext.reader().getNumericDocValues(VersionFieldMapper.NAME); } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + @Override public void process(HitContext hitContext) throws IOException { long version = Versions.NOT_FOUND; diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java index 8e062e47ca409..44e9a2a6e5193 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java @@ -19,11 +19,15 @@ import org.elasticsearch.search.fetch.FetchSearchResult; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; +import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.lookup.Source; import java.io.IOException; +import java.util.Collection; import java.util.HashMap; import java.util.Map; +import java.util.Set; public final class InnerHitsPhase implements FetchSubPhase { @@ -39,12 +43,18 @@ public FetchSubPhaseProcessor getProcessor(FetchContext searchContext) { return null; } Map innerHits = searchContext.innerHits().getInnerHits(); + StoredFieldsSpec storedFieldsSpec = new StoredFieldsSpec(requiresSource(innerHits.values()), false, Set.of()); return new FetchSubPhaseProcessor() { @Override public void setNextReader(LeafReaderContext readerContext) { } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return storedFieldsSpec; + } + @Override public void process(HitContext hitContext) throws IOException { SearchHit hit = hitContext.hit(); @@ -54,6 +64,16 @@ public void process(HitContext hitContext) throws IOException { }; } + private static boolean requiresSource(Collection subContexts) { + boolean requiresSource = false; + for (SearchContext sc : subContexts) { + requiresSource |= sc.sourceRequested(); + requiresSource |= sc.fetchFieldsContext() != null; + requiresSource |= sc.highlight() != null; + } + return requiresSource; + } + private void hitExecute(Map innerHits, SearchHit hit, Source rootSource) throws IOException { for (Map.Entry entry : innerHits.entrySet()) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java index 6ff1e698d96af..883bae9443bf3 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java @@ -17,6 +17,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.ArrayList; @@ -71,6 +72,11 @@ public void process(HitContext hitContext) { } hitContext.hit().matchedQueries(matches.toArray(String[]::new)); } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } }; } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsContext.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsContext.java index bcdfb2ed288cb..0725e6cb32fa6 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsContext.java @@ -39,7 +39,7 @@ public boolean ignoreException() { } } - private List fields = new ArrayList<>(); + private final List fields = new ArrayList<>(); public ScriptFieldsContext() {} diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsPhase.java index 1a84051a285dc..965ef59adb4f2 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsPhase.java @@ -14,12 +14,14 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Set; public final class ScriptFieldsPhase implements FetchSubPhase { @Override @@ -37,6 +39,15 @@ public void setNextReader(LeafReaderContext readerContext) { leafScripts = createLeafScripts(readerContext, scriptFields); } + @Override + public StoredFieldsSpec storedFieldsSpec() { + // If script fields need source then they will load it via SearchLookup, + // which has its own lazy loading config that kicks in if not overridden + // by other sub phases that require source. However, if script fields + // are present then we enforce metadata loading + return new StoredFieldsSpec(false, true, Set.of()); + } + @Override public void process(HitContext hitContext) { int docId = hitContext.docId(); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermPhase.java index ae606a87f60f5..13e26544af20a 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermPhase.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; @@ -34,6 +35,11 @@ public void setNextReader(LeafReaderContext readerContext) throws IOException { primaryTermField = readerContext.reader().getNumericDocValues(SeqNoFieldMapper.PRIMARY_TERM_NAME); } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + @Override public void process(HitContext hitContext) throws IOException { int docId = hitContext.docId(); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/StoredFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/StoredFieldsPhase.java new file mode 100644 index 0000000000000..d6950df962433 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/StoredFieldsPhase.java @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.fetch.subphase; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.index.mapper.IgnoredFieldMapper; +import org.elasticsearch.index.mapper.LegacyTypeFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.RoutingFieldMapper; +import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.fetch.FetchContext; +import org.elasticsearch.search.fetch.FetchSubPhase; +import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsContext; +import org.elasticsearch.search.fetch.StoredFieldsSpec; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Process stored fields loaded from a HitContext into DocumentFields + */ +public class StoredFieldsPhase implements FetchSubPhase { + + /** Associates a field name with a mapped field type and whether or not it is a metadata field */ + private record StoredField(String name, MappedFieldType ft, boolean isMetadataField) { + + /** Processes a set of stored fields using field type information */ + List process(Map> loadedFields) { + List inputs = loadedFields.get(ft.name()); + if (inputs == null) { + return List.of(); + } + return inputs.stream().map(ft::valueForDisplay).toList(); + } + + boolean hasValue(Map> loadedFields) { + return loadedFields.containsKey(ft.name()); + } + + } + + private static final List METADATA_FIELDS = List.of( + new StoredField("_routing", RoutingFieldMapper.FIELD_TYPE, true), + new StoredField("_ignored", IgnoredFieldMapper.FIELD_TYPE, true), + // pre-6.0 indexes can return a _type field, this will be valueless in modern indexes and ignored + new StoredField("_type", LegacyTypeFieldMapper.FIELD_TYPE, true) + ); + + @Override + public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) { + StoredFieldsContext storedFieldsContext = fetchContext.storedFieldsContext(); + if (storedFieldsContext == null || storedFieldsContext.fetchFields() == false) { + return null; + } + + // build the StoredFieldsSpec and a list of StoredField records to process + List storedFields = new ArrayList<>(METADATA_FIELDS); + Set fieldsToLoad = new HashSet<>(); + if (storedFieldsContext.fieldNames() != null) { + SearchExecutionContext sec = fetchContext.getSearchExecutionContext(); + for (String field : storedFieldsContext.fieldNames()) { + if (SourceFieldMapper.NAME.equals(field) == false) { + Collection fieldNames = sec.getMatchingFieldNames(field); + for (String fieldName : fieldNames) { + MappedFieldType ft = sec.getFieldType(fieldName); + if (ft.isStored() == false) { + continue; + } + storedFields.add(new StoredField(fieldName, ft, sec.isMetadataField(ft.name()))); + fieldsToLoad.add(ft.name()); + } + } + } + } + StoredFieldsSpec storedFieldsSpec = new StoredFieldsSpec(false, true, fieldsToLoad); + + return new FetchSubPhaseProcessor() { + @Override + public void setNextReader(LeafReaderContext readerContext) { + + } + + @Override + public void process(HitContext hitContext) { + Map> loadedFields = hitContext.loadedFields(); + Map docFields = new HashMap<>(); + Map metaFields = new HashMap<>(); + for (StoredField storedField : storedFields) { + if (storedField.hasValue(loadedFields)) { + DocumentField df = new DocumentField(storedField.name, storedField.process(loadedFields)); + if (storedField.isMetadataField) { + metaFields.put(storedField.name, df); + } else { + docFields.put(storedField.name, df); + } + } + } + hitContext.hit().addDocumentFields(docFields, metaFields); + } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return storedFieldsSpec; + } + }; + } + +} diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java index 4301e8734fcf9..4b60d32d3065f 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java @@ -16,12 +16,15 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import java.util.function.Function; public class HighlightPhase implements FetchSubPhase { @@ -43,12 +46,7 @@ public FetchSubPhaseProcessor getProcessor(FetchContext context) { public FetchSubPhaseProcessor getProcessor(FetchContext context, SearchHighlightContext highlightContext, Query query) { Map sharedCache = new HashMap<>(); - Map> contextBuilders = contextBuilders( - context, - highlightContext, - query, - sharedCache - ); + FieldContext fieldContext = contextBuilders(context, highlightContext, query, sharedCache); return new FetchSubPhaseProcessor() { @Override @@ -56,9 +54,15 @@ public void setNextReader(LeafReaderContext readerContext) { } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return fieldContext.storedFieldsSpec; + } + @Override public void process(HitContext hitContext) throws IOException { Map highlightFields = new HashMap<>(); + Map> contextBuilders = fieldContext.builders; for (String field : contextBuilders.keySet()) { FieldHighlightContext fieldContext = contextBuilders.get(field).apply(hitContext); Highlighter highlighter = getHighlighter(fieldContext.field); @@ -87,13 +91,16 @@ private Highlighter getHighlighter(SearchHighlightContext.Field field) { return highlighter; } - private Map> contextBuilders( + private record FieldContext(StoredFieldsSpec storedFieldsSpec, Map> builders) {} + + private FieldContext contextBuilders( FetchContext context, SearchHighlightContext highlightContext, Query query, Map sharedCache ) { Map> builders = new LinkedHashMap<>(); + StoredFieldsSpec storedFieldsSpec = StoredFieldsSpec.NO_REQUIREMENTS; for (SearchHighlightContext.Field field : highlightContext.fields()) { Highlighter highlighter = getHighlighter(field); @@ -104,8 +111,11 @@ private Map> contextBuilders throw new IllegalArgumentException("source is forced for fields " + fieldNamesToHighlight + " but _source is disabled"); } } + boolean forceSource = highlightContext.forceSource(field); boolean fieldNameContainsWildcards = field.field().contains("*"); + Set storedFields = new HashSet<>(); + boolean sourceRequired = forceSource; for (String fieldName : fieldNamesToHighlight) { MappedFieldType fieldType = context.getSearchExecutionContext().getFieldType(fieldName); @@ -129,9 +139,14 @@ private Map> contextBuilders } } + if (fieldType.isStored()) { + storedFields.add(fieldType.name()); + } else { + sourceRequired = true; + } + Query highlightQuery = field.fieldOptions().highlightQuery(); - boolean forceSource = highlightContext.forceSource(field); builders.put( fieldName, hc -> new FieldHighlightContext( @@ -146,7 +161,11 @@ private Map> contextBuilders ) ); } + // TODO in future we can load the storedFields in advance here and make use of them, + // but for now they are loaded separately in HighlightUtils so we only return whether + // or not we need source. + storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(sourceRequired, false, Set.of())); } - return builders; + return new FieldContext(storedFieldsSpec, builders); } } diff --git a/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java index 638dfec4927e9..79e762be7a61a 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java @@ -90,7 +90,7 @@ public void suggest(SuggestionSearchContext suggest) { @Override public boolean hasScriptFields() { - return scriptFields != null; + return scriptFields != null && scriptFields.fields().isEmpty() == false; } @Override diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index 01054e2b74230..0f2c55b586b7c 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -44,9 +44,7 @@ public void testCollapseSingleHit() throws IOException { List collapsedHits = new ArrayList<>(numInnerHits); for (int innerHitNum = 0; innerHitNum < numInnerHits; innerHitNum++) { SearchHits hits = new SearchHits( - new SearchHit[] { - new SearchHit(innerHitNum, "ID", Collections.emptyMap(), Collections.emptyMap()), - new SearchHit(innerHitNum + 1, "ID", Collections.emptyMap(), Collections.emptyMap()) }, + new SearchHit[] { new SearchHit(innerHitNum, "ID"), new SearchHit(innerHitNum + 1, "ID") }, new TotalHits(2, TotalHits.Relation.EQUAL_TO), 1.0F ); @@ -110,17 +108,9 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL } }; - SearchHits hits = new SearchHits( - new SearchHit[] { - new SearchHit( - 1, - "ID", - Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(collapseValue))), - Collections.emptyMap() - ) }, - new TotalHits(1, TotalHits.Relation.EQUAL_TO), - 1.0F - ); + SearchHit hit = new SearchHit(1, "ID"); + hit.setDocumentField("someField", new DocumentField("someField", Collections.singletonList(collapseValue))); + SearchHits hits = new SearchHits(new SearchHit[] { hit }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F); InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1); ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, internalSearchResponse, () -> new SearchPhase("test") { @Override @@ -147,9 +137,7 @@ public void testFailOneItemFailsEntirePhase() throws IOException { AtomicBoolean executedMultiSearch = new AtomicBoolean(false); SearchHits collapsedHits = new SearchHits( - new SearchHit[] { - new SearchHit(2, "ID", Collections.emptyMap(), Collections.emptyMap()), - new SearchHit(3, "ID", Collections.emptyMap(), Collections.emptyMap()) }, + new SearchHit[] { new SearchHit(2, "ID"), new SearchHit(3, "ID") }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F ); @@ -187,23 +175,11 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL } }; - SearchHits hits = new SearchHits( - new SearchHit[] { - new SearchHit( - 1, - "ID", - Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(collapseValue))), - Collections.emptyMap() - ), - new SearchHit( - 2, - "ID2", - Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(collapseValue))), - Collections.emptyMap() - ) }, - new TotalHits(1, TotalHits.Relation.EQUAL_TO), - 1.0F - ); + SearchHit hit1 = new SearchHit(1, "ID"); + hit1.setDocumentField("someField", new DocumentField("someField", Collections.singletonList(collapseValue))); + SearchHit hit2 = new SearchHit(2, "ID2"); + hit2.setDocumentField("someField", new DocumentField("someField", Collections.singletonList(collapseValue))); + SearchHits hits = new SearchHits(new SearchHit[] { hit1, hit2 }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F); InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1); ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, internalSearchResponse, () -> new SearchPhase("test") { @Override @@ -227,23 +203,11 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL } }; - SearchHits hits = new SearchHits( - new SearchHit[] { - new SearchHit( - 1, - "ID", - Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(null))), - Collections.emptyMap() - ), - new SearchHit( - 2, - "ID2", - Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(null))), - Collections.emptyMap() - ) }, - new TotalHits(1, TotalHits.Relation.EQUAL_TO), - 1.0F - ); + SearchHit hit1 = new SearchHit(1, "ID"); + hit1.setDocumentField("someField", new DocumentField("someField", Collections.singletonList(null))); + SearchHit hit2 = new SearchHit(2, "ID2"); + hit2.setDocumentField("someField", new DocumentField("someField", Collections.singletonList(null))); + SearchHits hits = new SearchHits(new SearchHit[] { hit1, hit2 }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F); InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1); ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, internalSearchResponse, () -> new SearchPhase("test") { @Override diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java index 5bfe68fe9637b..79fee3ac34ec1 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java @@ -422,7 +422,7 @@ private static AtomicArray generateFetchResults( List searchHits = new ArrayList<>(); for (ScoreDoc scoreDoc : mergedSearchDocs) { if (scoreDoc.shardIndex == shardIndex) { - searchHits.add(new SearchHit(scoreDoc.doc, "", Collections.emptyMap(), Collections.emptyMap())); + searchHits.add(new SearchHit(scoreDoc.doc, "")); if (scoreDoc.score > maxScore) { maxScore = scoreDoc.score; } @@ -433,7 +433,7 @@ private static AtomicArray generateFetchResults( for (CompletionSuggestion.Entry.Option option : ((CompletionSuggestion) suggestion).getOptions()) { ScoreDoc doc = option.getDoc(); if (doc.shardIndex == shardIndex) { - searchHits.add(new SearchHit(doc.doc, "", Collections.emptyMap(), Collections.emptyMap())); + searchHits.add(new SearchHit(doc.doc, "")); if (doc.score > maxScore) { maxScore = doc.score; } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java index 4815030e73a7b..cd45ddd680332 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java @@ -40,7 +40,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import static java.util.Collections.emptyList; @@ -216,7 +215,7 @@ public void testFromXContentWithFailures() throws IOException { } public void testToXContent() throws IOException { - SearchHit hit = new SearchHit(1, "id1", Collections.emptyMap(), Collections.emptyMap()); + SearchHit hit = new SearchHit(1, "id1"); hit.score(2.0f); SearchHit[] hits = new SearchHit[] { hit }; { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java index 3052bfdb3400d..52475e7b059b5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java @@ -19,7 +19,7 @@ public class IgnoredFieldTypeTests extends FieldTypeTestCase { public void testPrefixQuery() { - MappedFieldType ft = IgnoredFieldMapper.IgnoredFieldType.INSTANCE; + MappedFieldType ft = IgnoredFieldMapper.FIELD_TYPE; Query expected = new PrefixQuery(new Term("_ignored", new BytesRef("foo*"))); assertEquals(expected, ft.prefixQuery("foo*", null, MOCK_CONTEXT)); @@ -36,7 +36,7 @@ public void testPrefixQuery() { } public void testRegexpQuery() { - MappedFieldType ft = IgnoredFieldMapper.IgnoredFieldType.INSTANCE; + MappedFieldType ft = IgnoredFieldMapper.FIELD_TYPE; Query expected = new RegexpQuery(new Term("_ignored", new BytesRef("foo?"))); assertEquals(expected, ft.regexpQuery("foo?", 0, 0, 10, null, MOCK_CONTEXT)); @@ -49,7 +49,7 @@ public void testRegexpQuery() { } public void testWildcardQuery() { - MappedFieldType ft = IgnoredFieldMapper.IgnoredFieldType.INSTANCE; + MappedFieldType ft = IgnoredFieldMapper.FIELD_TYPE; Query expected = new WildcardQuery(new Term("_ignored", new BytesRef("foo*"))); assertEquals(expected, ft.wildcardQuery("foo*", null, MOCK_CONTEXT)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldTypeTests.java index 0d7659418d38e..234e5ff62d111 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldTypeTests.java @@ -18,7 +18,7 @@ public class RoutingFieldTypeTests extends FieldTypeTestCase { public void testPrefixQuery() { - MappedFieldType ft = RoutingFieldMapper.RoutingFieldType.INSTANCE; + MappedFieldType ft = RoutingFieldMapper.FIELD_TYPE; Query expected = new PrefixQuery(new Term("_routing", new BytesRef("foo*"))); assertEquals(expected, ft.prefixQuery("foo*", null, MOCK_CONTEXT)); @@ -35,7 +35,7 @@ public void testPrefixQuery() { } public void testRegexpQuery() { - MappedFieldType ft = RoutingFieldMapper.RoutingFieldType.INSTANCE; + MappedFieldType ft = RoutingFieldMapper.FIELD_TYPE; Query expected = new RegexpQuery(new Term("_routing", new BytesRef("foo?"))); assertEquals(expected, ft.regexpQuery("foo?", 0, 0, 10, null, MOCK_CONTEXT)); @@ -48,7 +48,7 @@ public void testRegexpQuery() { } public void testWildcardQuery() { - MappedFieldType ft = RoutingFieldMapper.RoutingFieldType.INSTANCE; + MappedFieldType ft = RoutingFieldMapper.FIELD_TYPE; Query expected = new WildcardQuery(new Term("_routing", new BytesRef("foo*"))); assertEquals(expected, ft.wildcardQuery("foo*", null, MOCK_CONTEXT)); diff --git a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java index 544c0794271a5..c50e0777bd6de 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java @@ -71,8 +71,8 @@ public static SearchHit createTestItem(XContentType xContentType, boolean withOp documentFields = GetResultTests.randomDocumentFields(xContentType, false).v2(); } } - - SearchHit hit = new SearchHit(internalId, uid, nestedIdentity, documentFields, metaFields); + SearchHit hit = new SearchHit(internalId, uid, nestedIdentity); + hit.addDocumentFields(documentFields, metaFields); if (frequently()) { if (rarely()) { hit.score(Float.NaN); @@ -212,7 +212,7 @@ public void testFromXContentWithoutTypeAndId() throws IOException { } public void testToXContent() throws IOException { - SearchHit searchHit = new SearchHit(1, "id1", Collections.emptyMap(), Collections.emptyMap()); + SearchHit searchHit = new SearchHit(1, "id1"); searchHit.score(1.5f); XContentBuilder builder = JsonXContent.contentBuilder(); searchHit.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -225,25 +225,25 @@ public void testSerializeShardTarget() throws Exception { SearchShardTarget target = new SearchShardTarget("_node_id", new ShardId(new Index("_index", "_na_"), 0), clusterAlias); Map innerHits = new HashMap<>(); - SearchHit innerHit1 = new SearchHit(0, "_id", null, null); + SearchHit innerHit1 = new SearchHit(0, "_id"); innerHit1.shard(target); - SearchHit innerInnerHit2 = new SearchHit(0, "_id", null, null); + SearchHit innerInnerHit2 = new SearchHit(0, "_id"); innerInnerHit2.shard(target); innerHits.put("1", new SearchHits(new SearchHit[] { innerInnerHit2 }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1f)); innerHit1.setInnerHits(innerHits); - SearchHit innerHit2 = new SearchHit(0, "_id", null, null); + SearchHit innerHit2 = new SearchHit(0, "_id"); innerHit2.shard(target); - SearchHit innerHit3 = new SearchHit(0, "_id", null, null); + SearchHit innerHit3 = new SearchHit(0, "_id"); innerHit3.shard(target); innerHits = new HashMap<>(); - SearchHit hit1 = new SearchHit(0, "_id", null, null); + SearchHit hit1 = new SearchHit(0, "_id"); innerHits.put("1", new SearchHits(new SearchHit[] { innerHit1, innerHit2 }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1f)); innerHits.put("2", new SearchHits(new SearchHit[] { innerHit3 }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1f)); hit1.shard(target); hit1.setInnerHits(innerHits); - SearchHit hit2 = new SearchHit(0, "_id", null, null); + SearchHit hit2 = new SearchHit(0, "_id"); hit2.shard(target); SearchHits hits = new SearchHits(new SearchHit[] { hit1, hit2 }, new TotalHits(2, TotalHits.Relation.EQUAL_TO), 1f); @@ -270,7 +270,7 @@ public void testSerializeShardTarget() throws Exception { } public void testNullSource() { - SearchHit searchHit = new SearchHit(0, "_id", null, null); + SearchHit searchHit = new SearchHit(0, "_id"); assertThat(searchHit.getSourceAsMap(), nullValue()); assertThat(searchHit.getSourceRef(), nullValue()); @@ -359,7 +359,8 @@ public void testToXContentEmptyFields() throws IOException { Map fields = new HashMap<>(); fields.put("foo", new DocumentField("foo", Collections.emptyList())); fields.put("bar", new DocumentField("bar", Collections.emptyList())); - SearchHit hit = new SearchHit(0, "_id", fields, Collections.emptyMap()); + SearchHit hit = new SearchHit(0, "_id"); + hit.addDocumentFields(fields, Map.of()); { BytesReference originalBytes = toShuffledXContent(hit, XContentType.JSON, ToXContent.EMPTY_PARAMS, randomBoolean()); // checks that the fields section is completely omitted in the rendering. @@ -377,7 +378,8 @@ public void testToXContentEmptyFields() throws IOException { fields = new HashMap<>(); fields.put("foo", new DocumentField("foo", Collections.emptyList())); fields.put("bar", new DocumentField("bar", Collections.singletonList("value"))); - hit = new SearchHit(0, "_id", fields, Collections.emptyMap()); + hit = new SearchHit(0, "_id"); + hit.addDocumentFields(fields, Collections.emptyMap()); { BytesReference originalBytes = toShuffledXContent(hit, XContentType.JSON, ToXContent.EMPTY_PARAMS, randomBoolean()); final SearchHit parsed; @@ -393,7 +395,8 @@ public void testToXContentEmptyFields() throws IOException { Map metadata = new HashMap<>(); metadata.put("_routing", new DocumentField("_routing", Collections.emptyList())); - hit = new SearchHit(0, "_id", fields, Collections.emptyMap()); + hit = new SearchHit(0, "_id"); + hit.addDocumentFields(fields, Collections.emptyMap()); { BytesReference originalBytes = toShuffledXContent(hit, XContentType.JSON, ToXContent.EMPTY_PARAMS, randomBoolean()); final SearchHit parsed; diff --git a/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java b/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java index 31f348015069f..528223fc40075 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; -import java.util.Collections; import java.util.function.Predicate; public class SearchHitsTests extends AbstractXContentSerializingTestCase { @@ -227,9 +226,7 @@ protected SearchHits doParseInstance(XContentParser parser) throws IOException { } public void testToXContent() throws IOException { - SearchHit[] hits = new SearchHit[] { - new SearchHit(1, "id1", Collections.emptyMap(), Collections.emptyMap()), - new SearchHit(2, "id2", Collections.emptyMap(), Collections.emptyMap()) }; + SearchHit[] hits = new SearchHit[] { new SearchHit(1, "id1"), new SearchHit(2, "id2") }; long totalHits = 1000; float maxScore = 1.5f; @@ -253,10 +250,7 @@ public void testToXContent() throws IOException { public void testFromXContentWithShards() throws IOException { for (boolean withExplanation : new boolean[] { true, false }) { - final SearchHit[] hits = new SearchHit[] { - new SearchHit(1, "id1", Collections.emptyMap(), Collections.emptyMap()), - new SearchHit(2, "id2", Collections.emptyMap(), Collections.emptyMap()), - new SearchHit(10, "id10", Collections.emptyMap(), Collections.emptyMap()) }; + final SearchHit[] hits = new SearchHit[] { new SearchHit(1, "id1"), new SearchHit(2, "id2"), new SearchHit(10, "id10") }; for (SearchHit hit : hits) { String index = randomAlphaOfLengthBetween(5, 10); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java index 0ef8081110df5..16c36697cfc3f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java @@ -154,7 +154,8 @@ private InternalTopHits createTestInstance( Map searchHitFields = new HashMap<>(); scoreDocs[i] = docBuilder.apply(docId, score); - hits[i] = new SearchHit(docId, Integer.toString(i), searchHitFields, Collections.emptyMap()); + hits[i] = new SearchHit(docId, Integer.toString(i)); + hits[i].addDocumentFields(searchHitFields, Collections.emptyMap()); hits[i].score(score); } int totalHits = between(actualSize, 500000); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java b/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java new file mode 100644 index 0000000000000..315c0673e168c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.fetch; + +import org.elasticsearch.script.Script; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.subphase.FetchSourcePhase; +import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext; +import org.elasticsearch.search.fetch.subphase.ScriptFieldsPhase; +import org.elasticsearch.search.fetch.subphase.StoredFieldsPhase; +import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.hasSize; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class StoredFieldsSpecTests extends ESTestCase { + + public void testDefaults() { + SearchSourceBuilder search = new SearchSourceBuilder(); + // defaults - return source and metadata fields + FetchContext fc = new FetchContext(searchContext(search)); + + FetchSubPhaseProcessor sourceProcessor = new FetchSourcePhase().getProcessor(fc); + assertNotNull(sourceProcessor); + StoredFieldsSpec sourceSpec = sourceProcessor.storedFieldsSpec(); + assertTrue(sourceSpec.requiresSource()); + assertThat(sourceSpec.requiredStoredFields(), hasSize(0)); + + FetchSubPhaseProcessor storedFieldsProcessor = new StoredFieldsPhase().getProcessor(fc); + assertNotNull(storedFieldsProcessor); + StoredFieldsSpec storedFieldsSpec = storedFieldsProcessor.storedFieldsSpec(); + assertFalse(storedFieldsSpec.requiresSource()); + assertThat(storedFieldsSpec.requiredStoredFields(), hasSize(0)); + + StoredFieldsSpec merged = sourceSpec.merge(storedFieldsSpec); + assertTrue(merged.requiresSource()); + assertThat(merged.requiredStoredFields(), hasSize(0)); + } + + public void testStoredFieldsDisabled() { + SearchSourceBuilder search = new SearchSourceBuilder(); + search.storedField("_none_"); + FetchContext fc = new FetchContext(searchContext(search)); + + assertNull(new StoredFieldsPhase().getProcessor(fc)); + assertNull(new FetchSourcePhase().getProcessor(fc)); + } + + public void testScriptFieldsEnableMetadata() { + SearchSourceBuilder search = new SearchSourceBuilder(); + search.scriptField("field", new Script("script")); + FetchContext fc = new FetchContext(searchContext(search)); + + FetchSubPhaseProcessor subPhaseProcessor = new ScriptFieldsPhase().getProcessor(fc); + assertNotNull(subPhaseProcessor); + StoredFieldsSpec spec = subPhaseProcessor.storedFieldsSpec(); + assertFalse(spec.requiresSource()); + assertTrue(spec.requiresMetadata()); + } + + private static SearchContext searchContext(SearchSourceBuilder sourceBuilder) { + SearchContext sc = mock(SearchContext.class); + when(sc.fetchSourceContext()).thenReturn(sourceBuilder.fetchSource()); + when(sc.storedFieldsContext()).thenReturn(sourceBuilder.storedFields()); + ScriptFieldsContext scriptContext = new ScriptFieldsContext(); + if (sourceBuilder.scriptFields() != null) { + for (SearchSourceBuilder.ScriptField scriptField : sourceBuilder.scriptFields()) { + scriptContext.add(new ScriptFieldsContext.ScriptField(scriptField.fieldName(), null, true)); + } + } + when(sc.scriptFields()).thenReturn(scriptContext); + return sc; + } + +} diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhaseTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhaseTests.java index 9ae94f385386f..e0a26fbc67ffd 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhaseTests.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.Set; import static org.mockito.ArgumentMatchers.any; @@ -82,7 +83,7 @@ public void testDocValueFetcher() throws IOException { processor.setNextReader(context); for (int doc = 0; doc < context.reader().maxDoc(); doc++) { SearchHit searchHit = new SearchHit(doc + context.docBase); - processor.process(new FetchSubPhase.HitContext(searchHit, context, doc, Source.empty(null))); + processor.process(new FetchSubPhase.HitContext(searchHit, context, doc, Map.of(), Source.empty(null))); assertNotNull(searchHit.getFields().get("field")); } } diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java index 20c736d9974aa..620706a01c88f 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java @@ -170,13 +170,13 @@ private HitContext hitExecuteMultiple( when(sec.isSourceEnabled()).thenReturn(sourceBuilder != null); when(fetchContext.getSearchExecutionContext()).thenReturn(sec); - final SearchHit searchHit = new SearchHit(1, null, nestedIdentity, null, null); + final SearchHit searchHit = new SearchHit(1, null, nestedIdentity); // We don't need a real index, just a LeafReaderContext which cannot be mocked. MemoryIndex index = new MemoryIndex(); LeafReaderContext leafReaderContext = index.createSearcher().getIndexReader().leaves().get(0); Source source = sourceBuilder == null ? Source.empty(null) : Source.fromBytes(BytesReference.bytes(sourceBuilder)); - HitContext hitContext = new HitContext(searchHit, leafReaderContext, 1, source); + HitContext hitContext = new HitContext(searchHit, leafReaderContext, 1, Map.of(), source); FetchSourcePhase phase = new FetchSourcePhase(); FetchSubPhaseProcessor processor = phase.getProcessor(fetchContext); diff --git a/test/framework/src/main/java/org/elasticsearch/search/fetch/HighlighterTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/fetch/HighlighterTestCase.java index 58288542d737c..11bbdbc9b1013 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/fetch/HighlighterTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/fetch/HighlighterTestCase.java @@ -63,9 +63,10 @@ protected final Map highlight(MapperService mapperServic FetchSubPhaseProcessor processor = highlightPhase.getProcessor(fetchContext(context, search)); Source source = Source.fromBytes(doc.source()); FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext( - new SearchHit(0, "id", null, null), + new SearchHit(0, "id"), ir.leaves().get(0), 0, + Map.of(), source ); processor.process(hitContext); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java index d7dfc3f7354d8..e2157345e025f 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; @@ -122,6 +123,11 @@ public List getFetchSubPhases(FetchPhaseConstructionContext conte @Override public void setNextReader(LeafReaderContext readerContext) {} + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + @Override public void process(FetchSubPhase.HitContext hitContext) { if (hitContext.hit().getId().startsWith("boom")) { diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java index 4e8651619c539..42487229a97c4 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java @@ -269,7 +269,7 @@ protected MultiSearchResponse shardOperation(Request request, ShardId shardId) t } return context.getFieldType(field); }); - final SearchHit hit = new SearchHit(scoreDoc.doc, visitor.id(), Map.of(), Map.of()); + final SearchHit hit = new SearchHit(scoreDoc.doc, visitor.id()); hit.sourceRef(filterSource(fetchSourceContext, visitor.source())); hits[j] = hit; } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java index 612a7f5c8efeb..06e823a949235 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java @@ -73,7 +73,7 @@ public void query(QueryRequest r, ActionListener l) { } long sortValue = implicitTiebreakerValues.get(ordinal); - SearchHit searchHit = new SearchHit(ordinal, String.valueOf(ordinal), null, null); + SearchHit searchHit = new SearchHit(ordinal, String.valueOf(ordinal)); searchHit.sortValues( new SearchSortValues( new Long[] { (long) ordinal, sortValue }, @@ -92,7 +92,7 @@ public void fetchHits(Iterable> refs, ActionListener ref : refs) { List hits = new ArrayList<>(ref.size()); for (HitReference hitRef : ref) { - hits.add(new SearchHit(-1, hitRef.id(), null, null)); + hits.add(new SearchHit(-1, hitRef.id())); } searchHits.add(hits); } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java index 336c4a5a18543..90c85c2637634 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java @@ -188,7 +188,8 @@ static class EventsAsHits { Map documentFields = new HashMap<>(); documentFields.put(KEY_FIELD_NAME, new DocumentField(KEY_FIELD_NAME, Collections.singletonList(value.v1()))); // save the timestamp both as docId (int) and as id (string) - SearchHit searchHit = new SearchHit(entry.getKey(), entry.getKey().toString(), documentFields, null); + SearchHit searchHit = new SearchHit(entry.getKey(), entry.getKey().toString()); + searchHit.addDocumentFields(documentFields, Map.of()); hits.add(searchHit); } } @@ -230,7 +231,7 @@ public void fetchHits(Iterable> refs, ActionListener ref : refs) { List hits = new ArrayList<>(ref.size()); for (HitReference hitRef : ref) { - hits.add(new SearchHit(-1, hitRef.id(), null, null)); + hits.add(new SearchHit(-1, hitRef.id())); } searchHits.add(hits); } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/CriterionOrdinalExtractionTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/CriterionOrdinalExtractionTests.java index b992ffe968fcb..224f2bcdca614 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/CriterionOrdinalExtractionTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/CriterionOrdinalExtractionTests.java @@ -26,7 +26,6 @@ import java.util.function.Supplier; import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.eql.EqlTestUtils.randomSearchLongSortValues; import static org.elasticsearch.xpack.eql.EqlTestUtils.randomSearchSortValues; @@ -152,7 +151,8 @@ private SearchHit searchHit(Object timeValue, Object tiebreakerValue, Supplier fields = new HashMap<>(); fields.put(tsField, new DocumentField(tsField, singletonList(timeValue))); fields.put(tbField, new DocumentField(tsField, singletonList(tiebreakerValue))); - SearchHit searchHit = new SearchHit(randomInt(), randomAlphaOfLength(10), fields, emptyMap()); + SearchHit searchHit = new SearchHit(randomInt(), randomAlphaOfLength(10)); + searchHit.addDocumentFields(fields, Map.of()); searchHit.sortValues(searchSortValues.get()); return searchHit; diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java index 7787f3e6ef171..475d8f23573f5 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java @@ -106,7 +106,7 @@ static class TestQueryClient implements QueryClient { @Override public void query(QueryRequest r, ActionListener l) { int ordinal = r.searchSource().terminateAfter(); - SearchHit searchHit = new SearchHit(ordinal, String.valueOf(ordinal), null, null); + SearchHit searchHit = new SearchHit(ordinal, String.valueOf(ordinal)); searchHit.sortValues( new SearchSortValues(new Long[] { (long) ordinal, 1L }, new DocValueFormat[] { DocValueFormat.RAW, DocValueFormat.RAW }) ); @@ -122,7 +122,7 @@ public void fetchHits(Iterable> refs, ActionListener ref : refs) { List hits = new ArrayList<>(ref.size()); for (HitReference hitRef : ref) { - hits.add(new SearchHit(-1, hitRef.id(), null, null)); + hits.add(new SearchHit(-1, hitRef.id())); } searchHits.add(hits); } @@ -386,7 +386,7 @@ private class SuccessfulESMockClient extends ESMockClient { @Override void handleSearchRequest(ActionListener listener, SearchRequest searchRequest) { int ordinal = searchRequest.source().terminateAfter(); - SearchHit searchHit = new SearchHit(ordinal, String.valueOf(ordinal), null, null); + SearchHit searchHit = new SearchHit(ordinal, String.valueOf(ordinal)); searchHit.sortValues( new SearchSortValues(new Long[] { (long) ordinal, 1L }, new DocValueFormat[] { DocValueFormat.RAW, DocValueFormat.RAW }) ); @@ -433,7 +433,7 @@ void handleSearchRequest(ActionListener void } private SearchHits prepareSearchHits() { - SearchHit hit1 = new SearchHit(0, "1", null, null); + SearchHit hit1 = new SearchHit(0, "1"); hit1.score(1f); hit1.shard(new SearchShardTarget("a", new ShardId("a", "indexUUID", 0), null)); - SearchHit hit2 = new SearchHit(0, "2", null, null); + SearchHit hit2 = new SearchHit(0, "2"); hit2.score(1f); hit2.shard(new SearchShardTarget("a", new ShardId("a", "indexUUID", 0), null)); - SearchHit hit3 = new SearchHit(0, "3*", null, null); + SearchHit hit3 = new SearchHit(0, "3*"); hit3.score(1f); hit3.shard(new SearchShardTarget("a", new ShardId("a", "indexUUID", 0), null)); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteForecastActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteForecastActionTests.java index c6ef6350441c8..e9129c450d56f 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteForecastActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteForecastActionTests.java @@ -85,6 +85,8 @@ private static SearchHit createForecastStatsHit(ForecastRequestStats.ForecastReq ForecastRequestStats.STATUS.getPreferredName(), new DocumentField(ForecastRequestStats.STATUS.getPreferredName(), Collections.singletonList(status.toString())) ); - return new SearchHit(0, "", documentFields, Collections.emptyMap()); + SearchHit hit = new SearchHit(0, ""); + hit.addDocumentFields(documentFields, Map.of()); + return hit; } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java index 9b6ab20af8f80..fcce9c2a38af2 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java @@ -540,7 +540,8 @@ private SearchResponse createSearchResponse(List timestamps, List fields.put(extractedFields.timeField(), new DocumentField("time", Collections.singletonList(timestamps.get(i)))); fields.put("field_1", new DocumentField("field_1", Collections.singletonList(field1Values.get(i)))); fields.put("field_2", new DocumentField("field_2", Collections.singletonList(field2Values.get(i)))); - SearchHit hit = new SearchHit(randomInt(), null, fields, null); + SearchHit hit = new SearchHit(randomInt(), null); + hit.addDocumentFields(fields, Map.of()); hits.add(hit); } SearchHits searchHits = new SearchHits(hits.toArray(new SearchHit[0]), new TotalHits(hits.size(), TotalHits.Relation.EQUAL_TO), 1); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoinerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoinerTests.java index e53b858fbf218..99dfd9e919a6a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoinerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoinerTests.java @@ -309,7 +309,7 @@ private void givenDataFrameBatches(List> batche } private static SearchHit newHit(String json) { - SearchHit hit = new SearchHit(randomInt(), randomAlphaOfLength(10), Collections.emptyMap(), Collections.emptyMap()); + SearchHit hit = new SearchHit(randomInt(), randomAlphaOfLength(10)); hit.sourceRef(new BytesArray(json)); return hit; } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java index d9bcf0ee0381a..e84a11c1be598 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java @@ -921,9 +921,9 @@ private static SearchResponse createSearchResponse(List> sou fields.put("field_1", new DocumentField("field_1", Collections.singletonList("foo"))); fields.put("field_2", new DocumentField("field_2", Collections.singletonList("foo"))); - SearchHit hit = new SearchHit(123, String.valueOf(map.hashCode()), fields, Collections.emptyMap()).sourceRef( - BytesReference.bytes(XContentFactory.jsonBuilder().map(_source)) - ); + SearchHit hit = new SearchHit(123, String.valueOf(map.hashCode())); + hit.addDocumentFields(fields, Collections.emptyMap()); + hit.sourceRef(BytesReference.bytes(XContentFactory.jsonBuilder().map(_source))); list.add(hit); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java index 43fbb70d86abb..d34343a4dc200 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java @@ -189,7 +189,8 @@ public MockClientBuilder prepareSearchFields(String indexName, List fields; public SearchHitBuilder(int docId) { - fields = new HashMap<>(); - hit = new SearchHit(docId, null, fields, null); + hit = new SearchHit(docId, null); } public SearchHitBuilder addField(String name, Object value) { @@ -34,7 +30,7 @@ public SearchHitBuilder addField(String name, Object value) { } public SearchHitBuilder addField(String name, List values) { - fields.put(name, new DocumentField(name, values)); + hit.setDocumentField(name, new DocumentField(name, values)); return this; } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java index 0d6c590268d15..cc7f2fdb8128f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java @@ -300,7 +300,7 @@ private SearchHit tokenHit(int idx, BytesReference source) { final Map accessToken = (Map) sourceMap.get("access_token"); @SuppressWarnings("unchecked") final Map userToken = (Map) accessToken.get("user_token"); - final SearchHit hit = new SearchHit(idx, "token_" + userToken.get("id"), null, null); + final SearchHit hit = new SearchHit(idx, "token_" + userToken.get("id")); hit.sourceRef(source); return hit; } catch (IOException e) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java index 609ca0cdaad0a..8d025666829b9 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java @@ -1013,7 +1013,7 @@ private void mockFindTokenFromRefreshToken(String refreshToken, UserToken userTo .realmRef(realmRef) .build(false); - final SearchHit hit = new SearchHit(randomInt(), "token_" + TokenService.hashTokenString(userToken.getId()), null, null); + final SearchHit hit = new SearchHit(randomInt(), "token_" + TokenService.hashTokenString(userToken.getId())); BytesReference source = TokenService.createTokenDocument(userToken, storedRefreshToken, clientAuthentication, Instant.now()); if (refreshTokenStatus != null) { var sourceAsMap = XContentHelper.convertToMap(source, false, XContentType.JSON).v2(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStoreTests.java index 96099e2fb1582..cb4279f15316d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStoreTests.java @@ -265,9 +265,7 @@ public void testFindTokensFor() { .mapToObj( i -> new SearchHit( randomIntBetween(0, Integer.MAX_VALUE), - SERVICE_ACCOUNT_TOKEN_DOC_TYPE + "-" + accountId.asPrincipal() + "/" + tokenNames[i], - Map.of(), - Map.of() + SERVICE_ACCOUNT_TOKEN_DOC_TYPE + "-" + accountId.asPrincipal() + "/" + tokenNames[i] ) ) .toArray(SearchHit[]::new); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java index e357a45c75df9..000b4f725f7ef 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java @@ -906,7 +906,7 @@ private SearchHit[] buildHits(List sourcePrivile final SearchHit[] hits = new SearchHit[sourcePrivileges.size()]; for (int i = 0; i < hits.length; i++) { final ApplicationPrivilegeDescriptor p = sourcePrivileges.get(i); - hits[i] = new SearchHit(i, "application-privilege_" + p.getApplication() + ":" + p.getName(), null, null); + hits[i] = new SearchHit(i, "application-privilege_" + p.getApplication() + ":" + p.getName()); hits[i].sourceRef(new BytesArray(Strings.toString(p))); } return hits; diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingExtractorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingExtractorTests.java index 3a8e5e5ea7a45..bdc9130955488 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingExtractorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingExtractorTests.java @@ -29,7 +29,6 @@ import java.util.function.Supplier; import static java.util.Collections.singletonList; -import static java.util.Collections.singletonMap; import static org.elasticsearch.xpack.ql.execution.search.extractor.AbstractFieldHitExtractor.MultiValueSupport.NONE; import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE; import static org.elasticsearch.xpack.ql.util.CollectionUtils.combine; @@ -84,7 +83,8 @@ public void testGet() { double value = randomDouble(); double expected = Math.log(value); DocumentField field = new DocumentField(fieldName, singletonList(value)); - SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField(fieldName, field); assertEquals(expected, extractor.process(hit)); } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java index bfeab90bcfdee..aca64a467934d 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java @@ -32,7 +32,6 @@ import static java.util.Arrays.asList; import static java.util.Collections.singletonList; -import static java.util.Collections.singletonMap; import static org.elasticsearch.common.time.DateUtils.toMilliSeconds; import static org.elasticsearch.xpack.ql.execution.search.extractor.AbstractFieldHitExtractor.MultiValueSupport.LENIENT; import static org.elasticsearch.xpack.ql.execution.search.extractor.AbstractFieldHitExtractor.MultiValueSupport.NONE; @@ -95,7 +94,8 @@ public void testGetDottedValueWithDocValues() { } DocumentField field = new DocumentField(fieldName, documentFieldValues); - SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField(fieldName, field); Object result = documentFieldValues.isEmpty() ? null : documentFieldValues.get(0); assertEquals(result, extractor.extract(hit)); } @@ -112,7 +112,8 @@ public void testGetDocValue() { documentFieldValues.add(randomValue()); } DocumentField field = new DocumentField(fieldName, documentFieldValues); - SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField(fieldName, field); Object result = documentFieldValues.isEmpty() ? null : documentFieldValues.get(0); assertEquals(result, extractor.extract(hit)); } @@ -126,7 +127,8 @@ public void testGetDate() { ZonedDateTime zdt = DateUtils.asDateTimeWithMillis(millis, zoneId).plusNanos(nanosOnly); List documentFieldValues = Collections.singletonList(StringUtils.toString(zdt)); DocumentField field = new DocumentField("my_date_nanos_field", documentFieldValues); - SearchHit hit = new SearchHit(1, null, singletonMap("my_date_nanos_field", field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField("my_date_nanos_field", field); FieldHitExtractor extractor = new FieldHitExtractor("my_date_nanos_field", DATETIME, zoneId, LENIENT); assertEquals(zdt, extractor.extract(hit)); } @@ -142,7 +144,8 @@ public void testMultiValuedDocValue() { String fieldName = randomAlphaOfLength(5); FieldHitExtractor fe = getFieldHitExtractor(fieldName); DocumentField field = new DocumentField(fieldName, asList("a", "b")); - SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField(fieldName, field); QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extract(hit)); assertThat(ex.getMessage(), is("Arrays (returned by [" + fieldName + "]) are not supported")); } @@ -151,7 +154,8 @@ public void testExtractSourcePath() { FieldHitExtractor fe = getFieldHitExtractor("a.b.c"); Object value = randomValue(); DocumentField field = new DocumentField("a.b.c", singletonList(value)); - SearchHit hit = new SearchHit(1, null, null, singletonMap("a.b.c", field), null); + SearchHit hit = new SearchHit(1, null, null); + hit.setDocumentField("a.b.c", field); assertThat(fe.extract(hit), is(value)); } @@ -159,7 +163,8 @@ public void testMultiValuedSource() { FieldHitExtractor fe = getFieldHitExtractor("a"); Object value = randomValue(); DocumentField field = new DocumentField("a", asList(value, value)); - SearchHit hit = new SearchHit(1, null, null, singletonMap("a", field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField("a", field); QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extract(hit)); assertThat(ex.getMessage(), is("Arrays (returned by [a]) are not supported")); } @@ -169,7 +174,8 @@ public void testMultiValuedSourceAllowed() { Object valueA = randomValue(); Object valueB = randomValue(); DocumentField field = new DocumentField("a", asList(valueA, valueB)); - SearchHit hit = new SearchHit(1, null, null, singletonMap("a", field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField("a", field); assertEquals(valueA, fe.extract(hit)); } @@ -181,7 +187,8 @@ public void testGeoShapeExtraction() { map.put("coordinates", asList(1d, 2d)); map.put("type", "Point"); DocumentField field = new DocumentField(fieldName, singletonList(map)); - SearchHit hit = new SearchHit(1, null, null, singletonMap(fieldName, field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField(fieldName, field); assertEquals(new GeoShape(1, 2), fe.extract(hit)); } @@ -197,18 +204,16 @@ public void testMultipleGeoShapeExtraction() { map2.put("coordinates", asList(3d, 4d)); map2.put("type", "Point"); DocumentField field = new DocumentField(fieldName, asList(map1, map2)); - SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField(fieldName, field); QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extract(hit)); assertThat(ex.getMessage(), is("Arrays (returned by [" + fieldName + "]) are not supported")); FieldHitExtractor lenientFe = new FieldHitExtractor(fieldName, randomBoolean() ? GEO_SHAPE : SHAPE, UTC, LENIENT); - assertEquals( - new GeoShape(3, 4), - lenientFe.extract( - new SearchHit(1, null, null, singletonMap(fieldName, new DocumentField(fieldName, singletonList(map2))), null) - ) - ); + SearchHit searchHit = new SearchHit(1, "1"); + searchHit.setDocumentField(fieldName, new DocumentField(fieldName, singletonList(map2))); + assertEquals(new GeoShape(3, 4), lenientFe.extract(searchHit)); } public void testUnsignedLongExtraction() { @@ -218,7 +223,8 @@ public void testUnsignedLongExtraction() { String fieldName = randomAlphaOfLength(10); DocumentField field = new DocumentField(fieldName, singletonList(value)); - SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField(fieldName, field); FieldHitExtractor fe = new FieldHitExtractor(fieldName, UNSIGNED_LONG, randomZone(), randomBoolean() ? NONE : LENIENT); assertEquals(bi, fe.extract(hit)); @@ -231,7 +237,8 @@ public void testVersionExtraction() { String fieldName = randomAlphaOfLength(10); DocumentField field = new DocumentField(fieldName, singletonList(value)); - SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField(fieldName, field); FieldHitExtractor fe = new FieldHitExtractor(fieldName, VERSION, randomZone(), randomBoolean() ? NONE : LENIENT); assertEquals(version.toString(), fe.extract(hit).toString()); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/TopHitsAggExtractorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/TopHitsAggExtractorTests.java index c816544a92e2b..b36f286645efc 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/TopHitsAggExtractorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/TopHitsAggExtractorTests.java @@ -116,19 +116,14 @@ public void testExtractUnsignedLong() { private SearchHits searchHitsOf(Object value) { TotalHits totalHits = new TotalHits(10, TotalHits.Relation.EQUAL_TO); - return new SearchHits( - new SearchHit[] { - new SearchHit( - 1, - "docId", - Collections.singletonMap("topHitsAgg", new DocumentField("field", Collections.singletonList(value))), - Collections.singletonMap( - "topHitsAgg", - new DocumentField("_ignored", Collections.singletonList(randomValueOtherThan(value, () -> randomAlphaOfLength(5)))) - ) - ) }, - totalHits, - 0.0f + SearchHit searchHit = new SearchHit(1, "docId"); + searchHit.addDocumentFields( + Collections.singletonMap("topHitsAgg", new DocumentField("field", Collections.singletonList(value))), + Collections.singletonMap( + "topHitsAgg", + new DocumentField("_ignored", Collections.singletonList(randomValueOtherThan(value, () -> randomAlphaOfLength(5)))) + ) ); + return new SearchHits(new SearchHit[] { searchHit }, totalHits, 0.0f); } } diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java index 9ab58fbc17b16..ac728f59d5d3b 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java @@ -86,7 +86,7 @@ public void testExecuteWithAggs() throws Exception { public void testExecuteAccessHits() throws Exception { CompareCondition condition = new CompareCondition("ctx.payload.hits.hits.0._score", CompareCondition.Op.EQ, 1, Clock.systemUTC()); - SearchHit hit = new SearchHit(0, "1", null, null); + SearchHit hit = new SearchHit(0, "1"); hit.score(1f); hit.shard(new SearchShardTarget("a", new ShardId("a", "indexUUID", 0), null)); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java index b81e0bc5651f5..6feb977255de9 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java @@ -208,7 +208,7 @@ void stopExecutor() {} SearchHit[] hits = new SearchHit[count]; for (int i = 0; i < count; i++) { String id = String.valueOf(i); - SearchHit hit = new SearchHit(1, id, Collections.emptyMap(), Collections.emptyMap()); + SearchHit hit = new SearchHit(1, id); hit.version(1L); hit.shard(new SearchShardTarget("nodeId", new ShardId(watchIndex, 0), "whatever")); hits[i] = hit; diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java index 22f4ec6d3a74c..348f552465c3a 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java @@ -225,7 +225,7 @@ public void testFindTriggeredWatchesGoodCase() { when(searchResponse1.getSuccessfulShards()).thenReturn(1); when(searchResponse1.getTotalShards()).thenReturn(1); BytesArray source = new BytesArray("{}"); - SearchHit hit = new SearchHit(0, "first_foo", null, null); + SearchHit hit = new SearchHit(0, "first_foo"); hit.version(1L); hit.shard(new SearchShardTarget("_node_id", new ShardId(index, 0), null)); hit.sourceRef(source); @@ -240,7 +240,7 @@ public void testFindTriggeredWatchesGoodCase() { }).when(client).execute(eq(SearchAction.INSTANCE), any(), any()); // First return a scroll response with a single hit and then with no hits - hit = new SearchHit(0, "second_foo", null, null); + hit = new SearchHit(0, "second_foo"); hit.version(1L); hit.shard(new SearchShardTarget("_node_id", new ShardId(index, 0), null)); hit.sourceRef(source);