From 547c8327b22b395293d02354e6c85d1f46995374 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 10 Nov 2022 08:40:22 +0000 Subject: [PATCH] Allow FetchSubPhaseProcessors to report their required stored fields (#91269) Loading of stored fields is currently handled directly in FetchPhase, with some fairly complex logic examining various bits of the FetchContext to work out what fields need to be loaded. This is further complicated by synthetic source, which may have its own stored field requirements. This commit tries to separate out these concerns a little by adding a new StoredFieldsSpec record that holds information about which stored fields need to be loaded. Each FetchSubPhaseProcessor can now report a StoredFieldsSpec detailing what its requirements are, and these specs can be merged together, along with requirements from a SourceLoader, to determine up-front what fields should be loaded by the StoredFieldLoader. The stored fields themselves are added into the SearchHit by a new StoredFieldsPhase, which handles alias resolution and value post- processing. The logic to determine when source should be loaded and when not, based on the presence of script fields or stored fields, is moved into FetchContext, which highlights some inconsistencies that can be fixed in follow-up commits. --- docs/reference/search/profile.asciidoc | 22 ++ .../mustache/SearchTemplateResponseTests.java | 3 +- modules/parent-join/build.gradle | 6 +- .../rest-api-spec/test/30_inner_hits.yml | 3 +- .../PercolatorHighlightSubFetchPhase.java | 9 +- .../PercolatorMatchedSlotSubFetchPhase.java | 6 + ...rcolatorMatchedSlotSubFetchPhaseTests.java | 7 +- .../DiscountedCumulativeGainTests.java | 6 +- .../rankeval/ExpectedReciprocalRankTests.java | 2 +- .../rankeval/MeanReciprocalRankTests.java | 2 +- .../index/rankeval/PrecisionAtKTests.java | 6 +- .../index/rankeval/RankEvalResponseTests.java | 2 +- .../index/rankeval/RatedSearchHitTests.java | 14 +- .../index/rankeval/RecallAtKTests.java | 4 +- .../reindex/AsyncBulkByScrollActionTests.java | 2 +- .../ClientScrollableHitSourceTests.java | 3 +- rest-api-spec/build.gradle | 3 + .../rest-api-spec/test/search/370_profile.yml | 21 +- .../action/search/TransportSearchIT.java | 6 + .../search/fetch/FetchSubPhasePluginIT.java | 5 + .../fieldvisitor/CustomFieldsVisitor.java | 13 +- .../index/get/ShardGetService.java | 8 +- .../index/mapper/IgnoredFieldMapper.java | 6 +- .../index/mapper/LegacyTypeFieldMapper.java | 6 +- .../index/mapper/RoutingFieldMapper.java | 6 +- .../search/DefaultSearchContext.java | 2 +- .../org/elasticsearch/search/SearchHit.java | 35 ++-- .../elasticsearch/search/SearchModule.java | 2 + .../search/fetch/FetchContext.java | 53 ++++- .../search/fetch/FetchPhase.java | 198 ++++-------------- .../search/fetch/FetchProfiler.java | 5 + .../search/fetch/FetchSubPhase.java | 11 +- .../search/fetch/FetchSubPhaseProcessor.java | 5 + .../search/fetch/StoredFieldsContext.java | 6 +- .../search/fetch/StoredFieldsSpec.java | 47 +++++ .../search/fetch/subphase/ExplainPhase.java | 6 + .../fetch/subphase/FetchDocValuesPhase.java | 6 + .../fetch/subphase/FetchFieldsPhase.java | 7 + .../fetch/subphase/FetchScorePhase.java | 6 + .../fetch/subphase/FetchSourcePhase.java | 8 +- .../fetch/subphase/FetchVersionPhase.java | 6 + .../search/fetch/subphase/InnerHitsPhase.java | 20 ++ .../fetch/subphase/MatchedQueriesPhase.java | 6 + .../fetch/subphase/ScriptFieldsContext.java | 2 +- .../fetch/subphase/ScriptFieldsPhase.java | 11 + .../fetch/subphase/SeqNoPrimaryTermPhase.java | 6 + .../fetch/subphase/StoredFieldsPhase.java | 122 +++++++++++ .../subphase/highlight/HighlightPhase.java | 37 +++- .../search/internal/SubSearchContext.java | 2 +- .../action/search/ExpandSearchPhaseTests.java | 66 ++---- .../search/SearchPhaseControllerTests.java | 4 +- .../action/search/SearchResponseTests.java | 3 +- .../index/mapper/IgnoredFieldTypeTests.java | 6 +- .../index/mapper/RoutingFieldTypeTests.java | 6 +- .../elasticsearch/search/SearchHitTests.java | 29 +-- .../elasticsearch/search/SearchHitsTests.java | 10 +- .../metrics/InternalTopHitsTests.java | 3 +- .../search/fetch/StoredFieldsSpecTests.java | 83 ++++++++ .../fetch/subphase/FetchFieldsPhaseTests.java | 3 +- .../fetch/subphase/FetchSourcePhaseTests.java | 4 +- .../search/fetch/HighlighterTestCase.java | 3 +- .../search/AsyncSearchSingleNodeTests.java | 6 + .../action/EnrichShardMultiSearchAction.java | 2 +- .../assembler/ImplicitTiebreakerTests.java | 4 +- .../assembler/SequenceSpecTests.java | 5 +- .../CriterionOrdinalExtractionTests.java | 4 +- .../sequence/CircuitBreakerTests.java | 8 +- .../TransportGetPipelineActionTests.java | 6 +- .../TransportDeleteForecastActionTests.java | 4 +- .../scroll/ScrollDataExtractorTests.java | 3 +- .../process/DataFrameRowsJoinerTests.java | 2 +- .../persistence/JobResultsProviderTests.java | 6 +- .../ml/job/persistence/MockClientBuilder.java | 3 +- .../xpack/ml/test/SearchHitBuilder.java | 8 +- ...sportSamlInvalidateSessionActionTests.java | 2 +- .../security/authc/TokenServiceTests.java | 2 +- .../IndexServiceAccountTokenStoreTests.java | 4 +- .../store/NativePrivilegeStoreTests.java | 2 +- .../extractor/ComputingExtractorTests.java | 4 +- .../extractor/FieldHitExtractorTests.java | 43 ++-- .../extractor/TopHitsAggExtractorTests.java | 21 +- .../CompareConditionSearchTests.java | 2 +- .../xpack/watcher/WatcherServiceTests.java | 2 +- .../execution/TriggeredWatchStoreTests.java | 4 +- 84 files changed, 738 insertions(+), 409 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java create mode 100644 server/src/main/java/org/elasticsearch/search/fetch/subphase/StoredFieldsPhase.java create mode 100644 server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc index c8793e06f02a2..30a554ec0b60a 100644 --- a/docs/reference/search/profile.asciidoc +++ b/docs/reference/search/profile.asciidoc @@ -193,6 +193,17 @@ The API returns the following result: "debug": { "fast_path": 5 } + }, + { + "type": "StoredFieldsPhase", + "description": "", + "time_in_nanos": 5310, + "breakdown": { + "next_reader": 745, + "next_reader_count": 1, + "process": 4445, + "process_count": 5 + } } ] } @@ -1011,6 +1022,17 @@ And here is the fetch profile: "debug": { "fast_path": 4 } + }, + { + "type": "StoredFieldsPhase", + "description": "", + "time_in_nanos": 5310, + "breakdown": { + "next_reader": 745, + "next_reader_count": 1, + "process": 4445, + "process_count": 5 + } } ] } diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java index dd0739abbd938..d3f23d3f4a21c 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.xcontent.XContentType; import java.io.IOException; -import java.util.Collections; import java.util.function.Predicate; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; @@ -158,7 +157,7 @@ public void testSourceToXContent() throws IOException { } public void testSearchResponseToXContent() throws IOException { - SearchHit hit = new SearchHit(1, "id", Collections.emptyMap(), Collections.emptyMap()); + SearchHit hit = new SearchHit(1, "id"); hit.score(2.0f); SearchHit[] hits = new SearchHit[] { hit }; diff --git a/modules/parent-join/build.gradle b/modules/parent-join/build.gradle index 9704d35d80917..66079cfd444cb 100644 --- a/modules/parent-join/build.gradle +++ b/modules/parent-join/build.gradle @@ -18,4 +18,8 @@ restResources { restApi { include '_common', 'bulk', 'cluster', 'nodes', 'indices', 'index', 'search' } -} \ No newline at end of file +} + +tasks.named("yamlRestTestV7CompatTransform").configure { task -> + task.skipTest("/30_inner_hits/profile fetch", "profile output has changed") +} diff --git a/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/30_inner_hits.yml b/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/30_inner_hits.yml index 48cb6453b17bd..fab161baf1750 100644 --- a/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/30_inner_hits.yml +++ b/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/30_inner_hits.yml @@ -141,7 +141,7 @@ profile fetch: - gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 } - gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 } - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] } - - length: { profile.shards.0.fetch.children: 2 } + - length: { profile.shards.0.fetch.children: 3 } - match: { profile.shards.0.fetch.children.0.type: FetchSourcePhase } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 } @@ -152,3 +152,4 @@ profile fetch: - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 } - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 } - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 } + - match: { profile.shards.0.fetch.children.2.type: StoredFieldsPhase } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java index fd7a3b940bde9..e8d6edfd25a7b 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java @@ -18,6 +18,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.fetch.subphase.highlight.HighlightField; import org.elasticsearch.search.fetch.subphase.highlight.HighlightPhase; import org.elasticsearch.search.fetch.subphase.highlight.Highlighter; @@ -59,6 +60,11 @@ public void setNextReader(LeafReaderContext readerContext) { this.ctx = readerContext; } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + @Override public void process(HitContext hit) throws IOException { boolean singlePercolateQuery = percolateQueries.size() == 1; @@ -83,9 +89,10 @@ public void process(HitContext hit) throws IOException { int slot = (int) matchedSlot; BytesReference document = percolateQuery.getDocuments().get(slot); HitContext subContext = new HitContext( - new SearchHit(slot, "unknown", Collections.emptyMap(), Collections.emptyMap()), + new SearchHit(slot, "unknown"), percolatorLeafReaderContext, slot, + Map.of(), Source.fromBytes(document) ); // force source because MemoryIndex does not store fields diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java index a25d3c09e56bf..45c9b13fbb603 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java @@ -26,6 +26,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.ArrayList; @@ -69,6 +70,11 @@ public void setNextReader(LeafReaderContext readerContext) { this.ctx = readerContext; } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + @Override public void process(HitContext hitContext) throws IOException { for (PercolateContext pc : percolateContexts) { diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java index 8b4c1b5a6bddd..15b77ad304c54 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java @@ -34,6 +34,7 @@ import org.elasticsearch.test.ESTestCase; import java.util.Collections; +import java.util.Map; import java.util.stream.IntStream; import static org.mockito.Mockito.mock; @@ -55,7 +56,7 @@ public void testHitsExecute() throws Exception { LeafReaderContext context = reader.leaves().get(0); // A match: { - HitContext hit = new HitContext(new SearchHit(0), context, 0, Source.empty(null)); + HitContext hit = new HitContext(new SearchHit(0), context, 0, Map.of(), Source.empty(null)); PercolateQuery.QueryStore queryStore = ctx -> docId -> new TermQuery(new Term("field", "value")); MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "value", new WhitespaceAnalyzer()); @@ -86,7 +87,7 @@ public void testHitsExecute() throws Exception { // No match: { - HitContext hit = new HitContext(new SearchHit(0), context, 0, Source.empty(null)); + HitContext hit = new HitContext(new SearchHit(0), context, 0, Map.of(), Source.empty(null)); PercolateQuery.QueryStore queryStore = ctx -> docId -> new TermQuery(new Term("field", "value")); MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "value1", new WhitespaceAnalyzer()); @@ -116,7 +117,7 @@ public void testHitsExecute() throws Exception { // No query: { - HitContext hit = new HitContext(new SearchHit(0), context, 0, Source.empty(null)); + HitContext hit = new HitContext(new SearchHit(0), context, 0, Map.of(), Source.empty(null)); PercolateQuery.QueryStore queryStore = ctx -> docId -> null; MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "value", new WhitespaceAnalyzer()); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java index 7a8ebcd8d5ae7..d761b8e28aa95 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java @@ -60,7 +60,7 @@ public void testDCGAt() { SearchHit[] hits = new SearchHit[6]; for (int i = 0; i < 6; i++) { rated.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i])); - hits[i] = new SearchHit(i, Integer.toString(i), Collections.emptyMap(), Collections.emptyMap()); + hits[i] = new SearchHit(i, Integer.toString(i)); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); @@ -110,7 +110,7 @@ public void testDCGAtSixMissingRatings() { rated.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i])); } } - hits[i] = new SearchHit(i, Integer.toString(i), Collections.emptyMap(), Collections.emptyMap()); + hits[i] = new SearchHit(i, Integer.toString(i)); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); @@ -167,7 +167,7 @@ public void testDCGAtFourMoreRatings() { // only create four hits SearchHit[] hits = new SearchHit[4]; for (int i = 0; i < 4; i++) { - hits[i] = new SearchHit(i, Integer.toString(i), Collections.emptyMap(), Collections.emptyMap()); + hits[i] = new SearchHit(i, Integer.toString(i)); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); } DiscountedCumulativeGain dcg = new DiscountedCumulativeGain(); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java index 3cb873b4c22ff..79937c5f54604 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java @@ -103,7 +103,7 @@ private SearchHit[] createSearchHits(List rated, Integer[] releva if (relevanceRatings[i] != null) { rated.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i])); } - hits[i] = new SearchHit(i, Integer.toString(i), Collections.emptyMap(), Collections.emptyMap()); + hits[i] = new SearchHit(i, Integer.toString(i)); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); } return hits; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java index 04adb1f66b830..73a2eba86345a 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java @@ -190,7 +190,7 @@ public void testXContentParsingIsNotLenient() throws IOException { private static SearchHit[] createSearchHits(int from, int to, String index) { SearchHit[] hits = new SearchHit[to + 1 - from]; for (int i = from; i <= to; i++) { - hits[i] = new SearchHit(i, i + "", Collections.emptyMap(), Collections.emptyMap()); + hits[i] = new SearchHit(i, i + ""); hits[i].shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null)); } return hits; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java index 092aafbe2b484..306b6cafd4c9d 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java @@ -100,7 +100,7 @@ public void testIgnoreUnlabeled() { rated.add(createRatedDoc("test", "1", RELEVANT_RATING)); // add an unlabeled search hit SearchHit[] searchHits = Arrays.copyOf(toSearchHits(rated, "test"), 3); - searchHits[2] = new SearchHit(2, "2", Collections.emptyMap(), Collections.emptyMap()); + searchHits[2] = new SearchHit(2, "2"); searchHits[2].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", searchHits, rated); @@ -119,7 +119,7 @@ public void testIgnoreUnlabeled() { public void testNoRatedDocs() throws Exception { SearchHit[] hits = new SearchHit[5]; for (int i = 0; i < 5; i++) { - hits[i] = new SearchHit(i, i + "", Collections.emptyMap(), Collections.emptyMap()); + hits[i] = new SearchHit(i, i + ""); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); } EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", hits, Collections.emptyList()); @@ -248,7 +248,7 @@ private static PrecisionAtK mutate(PrecisionAtK original) { private static SearchHit[] toSearchHits(List rated, String index) { SearchHit[] hits = new SearchHit[rated.size()]; for (int i = 0; i < rated.size(); i++) { - hits[i] = new SearchHit(i, i + "", Collections.emptyMap(), Collections.emptyMap()); + hits[i] = new SearchHit(i, i + ""); hits[i].shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null)); } return hits; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java index e52f466b78e15..b602f2b0a119d 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java @@ -197,7 +197,7 @@ public void testToXContent() throws IOException { } private static RatedSearchHit searchHit(String index, int docId, Integer rating) { - SearchHit hit = new SearchHit(docId, docId + "", Collections.emptyMap(), Collections.emptyMap()); + SearchHit hit = new SearchHit(docId, docId + ""); hit.shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null)); hit.score(1.0f); return new RatedSearchHit(hit, rating != null ? OptionalInt.of(rating) : OptionalInt.empty()); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java index d441c96cb4328..d6cfb21049969 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java @@ -26,12 +26,7 @@ public class RatedSearchHitTests extends ESTestCase { public static RatedSearchHit randomRatedSearchHit() { OptionalInt rating = randomBoolean() ? OptionalInt.empty() : OptionalInt.of(randomIntBetween(0, 5)); - SearchHit searchHit = new SearchHit( - randomIntBetween(0, 10), - randomAlphaOfLength(10), - Collections.emptyMap(), - Collections.emptyMap() - ); + SearchHit searchHit = new SearchHit(randomIntBetween(0, 10), randomAlphaOfLength(10)); RatedSearchHit ratedSearchHit = new RatedSearchHit(searchHit, rating); return ratedSearchHit; } @@ -41,12 +36,7 @@ private static RatedSearchHit mutateTestItem(RatedSearchHit original) { SearchHit hit = original.getSearchHit(); switch (randomIntBetween(0, 1)) { case 0 -> rating = rating.isPresent() ? OptionalInt.of(rating.getAsInt() + 1) : OptionalInt.of(randomInt(5)); - case 1 -> hit = new SearchHit( - hit.docId(), - hit.getId() + randomAlphaOfLength(10), - Collections.emptyMap(), - Collections.emptyMap() - ); + case 1 -> hit = new SearchHit(hit.docId(), hit.getId() + randomAlphaOfLength(10)); default -> throw new IllegalStateException("The test should only allow two parameters mutated"); } return new RatedSearchHit(hit, rating); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RecallAtKTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RecallAtKTests.java index d021b0a6037e6..866675f4a9c11 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RecallAtKTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RecallAtKTests.java @@ -102,7 +102,7 @@ public void testNoRatedDocs() throws Exception { int k = 5; SearchHit[] hits = new SearchHit[k]; for (int i = 0; i < k; i++) { - hits[i] = new SearchHit(i, i + "", Collections.emptyMap(), Collections.emptyMap()); + hits[i] = new SearchHit(i, i + ""); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null)); } @@ -216,7 +216,7 @@ private static RecallAtK mutate(RecallAtK original) { private static SearchHit[] toSearchHits(List rated, String index) { SearchHit[] hits = new SearchHit[rated.size()]; for (int i = 0; i < rated.size(); i++) { - hits[i] = new SearchHit(i, i + "", Collections.emptyMap(), Collections.emptyMap()); + hits[i] = new SearchHit(i, i + ""); hits[i].shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null)); } return hits; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java index ff96bab7134c1..cbab4f892b215 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java @@ -565,7 +565,7 @@ protected RequestWrapper buildRequest(Hit doc) { action.start(); // create a simulated response. - SearchHit hit = new SearchHit(0, "id", emptyMap(), emptyMap()).sourceRef(new BytesArray("{}")); + SearchHit hit = new SearchHit(0, "id").sourceRef(new BytesArray("{}")); SearchHits hits = new SearchHits( IntStream.range(0, 100).mapToObj(i -> hit).toArray(SearchHit[]::new), new TotalHits(0, TotalHits.Relation.EQUAL_TO), diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java index da00f8b44036d..179d301f18dbc 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java @@ -46,7 +46,6 @@ import java.util.function.Function; import java.util.stream.IntStream; -import static java.util.Collections.emptyMap; import static org.apache.lucene.tests.util.TestUtil.randomSimpleString; import static org.elasticsearch.core.TimeValue.timeValueSeconds; import static org.hamcrest.Matchers.instanceOf; @@ -160,7 +159,7 @@ public void testScrollKeepAlive() { private SearchResponse createSearchResponse() { // create a simulated response. - SearchHit hit = new SearchHit(0, "id", emptyMap(), emptyMap()).sourceRef(new BytesArray("{}")); + SearchHit hit = new SearchHit(0, "id").sourceRef(new BytesArray("{}")); SearchHits hits = new SearchHits( IntStream.range(0, randomIntBetween(0, 20)).mapToObj(i -> hit).toArray(SearchHit[]::new), new TotalHits(0, TotalHits.Relation.EQUAL_TO), diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index ddf78b4301ae8..e92d6767a8032 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -75,6 +75,9 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task -> task.skipTest("search_shards/10_basic/Search shards aliases with and without filters", "Filter representation no longer outputs default boosts") task.skipTest("migration/10_get_feature_upgrade_status/Get feature upgrade status", "Awaits backport") task.skipTest("search/330_fetch_fields/Test disable source", "Error no longer thrown") + task.skipTest("search/370_profile/fetch fields", "profile output has changed") + task.skipTest("search/370_profile/fetch source", "profile output has changed") + task.skipTest("search/370_profile/fetch nested source", "profile output has changed") task.skipTest("search/240_date_nanos/doc value fields are working as expected across date and date_nanos fields", "Fetching docvalues field multiple times is no longer allowed") task.replaceValueInMatch("_type", "_doc") diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml index 9b9fe18276339..415da2ad8767b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml @@ -22,8 +22,8 @@ setup: --- fetch fields: - skip: - version: ' - 7.15.99' - reason: fetch profiling implemented in 7.16.0 + version: ' - 8.5.99' + reason: stored fields phase added in 8.6 - do: search: @@ -42,8 +42,9 @@ fetch fields: - gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 } - gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 } - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] } - - length: { profile.shards.0.fetch.children: 1 } + - length: { profile.shards.0.fetch.children: 2 } - match: { profile.shards.0.fetch.children.0.type: FetchFieldsPhase } + - match: { profile.shards.0.fetch.children.1.type: StoredFieldsPhase } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 } @@ -52,8 +53,8 @@ fetch fields: --- fetch source: - skip: - version: ' - 7.99.99' - reason: fetch profiling implemented in 8.0.0 to be backported to 7.16.0 + version: ' - 8.5.99' + reason: stored fields phase added in 8.6 - do: search: @@ -70,19 +71,20 @@ fetch source: - gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 } - gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 } - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] } - - length: { profile.shards.0.fetch.children: 1 } + - length: { profile.shards.0.fetch.children: 2 } - match: { profile.shards.0.fetch.children.0.type: FetchSourcePhase } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 } - match: { profile.shards.0.fetch.children.0.debug.fast_path: 1 } + - match: { profile.shards.0.fetch.children.1.type: StoredFieldsPhase } --- fetch nested source: - skip: - version: ' - 7.15.99' - reason: fetch profiling implemented in 7.16.0 + version: ' - 8.5.99' + reason: stored fields phase added in 8.6 - do: indices.create: @@ -133,7 +135,7 @@ fetch nested source: - gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 } - gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 } - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] } - - length: { profile.shards.0.fetch.children: 2 } + - length: { profile.shards.0.fetch.children: 3 } - match: { profile.shards.0.fetch.children.0.type: FetchSourcePhase } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 } @@ -144,6 +146,7 @@ fetch nested source: - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 } - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 } - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 } + - match: { profile.shards.0.fetch.children.2.type: StoredFieldsPhase } --- disabling stored fields removes fetch sub phases: diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java index 561d61185c9d5..2d772e25131c0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java @@ -56,6 +56,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xcontent.ObjectParser; @@ -92,6 +93,11 @@ public List getFetchSubPhases(FetchPhaseConstructionContext conte @Override public void setNextReader(LeafReaderContext readerContext) {} + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + @Override public void process(FetchSubPhase.HitContext hitContext) { if (fetchContext.getIndexName().startsWith("boom")) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java index 4b19abf28a5f2..e2c295985c3bc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java @@ -117,6 +117,11 @@ public void setNextReader(LeafReaderContext readerContext) { } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + @Override public void process(HitContext hitContext) throws IOException { hitExecute(searchContext, hitContext); diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/CustomFieldsVisitor.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/CustomFieldsVisitor.java index 6e2835488bf10..d7f6e3541838b 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/CustomFieldsVisitor.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/CustomFieldsVisitor.java @@ -10,13 +10,12 @@ import org.apache.lucene.index.FieldInfo; import java.util.HashSet; +import java.util.List; import java.util.Set; /** - * A field visitor that allows to load a selection of the stored fields by exact name or by pattern. - * Supported pattern styles: "xxx*", "*xxx", "*xxx*" and "xxx*yyy". - * The Uid field is always loaded. - * The class is optimized for source loading as it is a common use case. + * A field visitor that allows to load a selection of the stored fields by exact name + * {@code _id} and {@code _routing} fields are always loaded. */ public class CustomFieldsVisitor extends FieldsVisitor { @@ -24,7 +23,11 @@ public class CustomFieldsVisitor extends FieldsVisitor { public CustomFieldsVisitor(Set fields, boolean loadSource) { super(loadSource); - this.fields = fields; + this.fields = new HashSet<>(fields); + // metadata fields are already handled by FieldsVisitor, so removing + // them here means that if the only fields requested are metadata + // fields then we can shortcut loading + List.of("_id", "_routing", "_source").forEach(this.fields::remove); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 71177cfdff77e..b16715b8769d7 100644 --- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; @@ -29,7 +30,6 @@ import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.search.fetch.FetchPhase; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.lookup.Source; @@ -267,7 +267,11 @@ private GetResult innerGetFetch( if (false == needed.contains(entry.getKey())) { continue; } - List values = FetchPhase.processStoredField(mapperService::fieldType, entry.getKey(), entry.getValue()); + MappedFieldType ft = mapperService.fieldType(entry.getKey()); + if (ft == null) { + continue; // user asked for a non-existent field, ignore it + } + List values = entry.getValue().stream().map(ft::valueForDisplay).toList(); if (mapperService.isMetadataField(entry.getKey())) { metadataFields.put(entry.getKey(), new DocumentField(entry.getKey(), values)); } else { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java index d65c5897e54fd..ecbeff9d8cb41 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java @@ -40,14 +40,14 @@ public static class Defaults { } } + public static final IgnoredFieldType FIELD_TYPE = new IgnoredFieldType(); + private static final IgnoredFieldMapper INSTANCE = new IgnoredFieldMapper(); public static final TypeParser PARSER = new FixedTypeParser(c -> INSTANCE); public static final class IgnoredFieldType extends StringFieldType { - public static final IgnoredFieldType INSTANCE = new IgnoredFieldType(); - private IgnoredFieldType() { super(NAME, true, true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); } @@ -73,7 +73,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) } private IgnoredFieldMapper() { - super(IgnoredFieldType.INSTANCE); + super(FIELD_TYPE); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java index ac800e147d44e..a6075b9a46627 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java @@ -29,6 +29,8 @@ public class LegacyTypeFieldMapper extends MetadataFieldMapper { public static final String CONTENT_TYPE = "_type"; + public static final MappedFieldType FIELD_TYPE = new LegacyTypeFieldType(); + private static final LegacyTypeFieldMapper INSTANCE = new LegacyTypeFieldMapper(); public static final TypeParser PARSER = new FixedTypeParser(c -> INSTANCE); @@ -36,7 +38,7 @@ public class LegacyTypeFieldMapper extends MetadataFieldMapper { private static final Map ANALYZERS = Map.of(NAME, Lucene.KEYWORD_ANALYZER); protected LegacyTypeFieldMapper() { - super(new LegacyTypeFieldType()); + super(FIELD_TYPE); } @Override @@ -44,7 +46,7 @@ public Map indexAnalyzers() { return ANALYZERS; } - static final class LegacyTypeFieldType extends TermBasedFieldType { + private static final class LegacyTypeFieldType extends TermBasedFieldType { LegacyTypeFieldType() { super(NAME, false, true, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java index 4a38289ec3202..f7ef8b9113cbc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java @@ -67,9 +67,9 @@ public RoutingFieldMapper build() { public static final TypeParser PARSER = new ConfigurableTypeParser(c -> RoutingFieldMapper.get(Defaults.REQUIRED), c -> new Builder()); - static final class RoutingFieldType extends StringFieldType { + public static final MappedFieldType FIELD_TYPE = new RoutingFieldType(); - static RoutingFieldType INSTANCE = new RoutingFieldType(); + static final class RoutingFieldType extends StringFieldType { private RoutingFieldType() { super(NAME, true, true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); @@ -101,7 +101,7 @@ public static RoutingFieldMapper get(boolean required) { } private RoutingFieldMapper(boolean required) { - super(RoutingFieldType.INSTANCE); + super(FIELD_TYPE); this.required = required; } diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 5ec5a38978c73..e8b23a522b050 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -404,7 +404,7 @@ public void addRescore(RescoreContext rescore) { @Override public boolean hasScriptFields() { - return scriptFields != null; + return scriptFields != null && scriptFields.fields().isEmpty() == false; } @Override diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java index b5f3164987e86..3ef0e0e6d95d5 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHit.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java @@ -85,8 +85,8 @@ public final class SearchHit implements Writeable, ToXContentObject, Iterable documentFields; - private final Map metaFields; + private final Map documentFields = new HashMap<>(); + private final Map metaFields = new HashMap<>(); private Map highlightFields = null; @@ -111,20 +111,14 @@ public final class SearchHit implements Writeable, ToXContentObject, Iterable documentFields, Map metaFields) { - this(docId, id, null, documentFields, metaFields); + public SearchHit(int docId, String id) { + this(docId, id, null); } - public SearchHit( - int nestedTopDocId, - String id, - NestedIdentity nestedIdentity, - Map documentFields, - Map metaFields - ) { + public SearchHit(int nestedTopDocId, String id, NestedIdentity nestedIdentity) { this.docId = nestedTopDocId; if (id != null) { this.id = new Text(id); @@ -132,8 +126,6 @@ public SearchHit( this.id = null; } this.nestedIdentity = nestedIdentity; - this.documentFields = documentFields == null ? emptyMap() : documentFields; - this.metaFields = metaFields == null ? emptyMap() : metaFields; } public SearchHit(StreamInput in) throws IOException { @@ -155,12 +147,10 @@ public SearchHit(StreamInput in) throws IOException { explanation = readExplanation(in); } if (in.getVersion().onOrAfter(Version.V_7_8_0)) { - documentFields = in.readMap(StreamInput::readString, DocumentField::new); - metaFields = in.readMap(StreamInput::readString, DocumentField::new); + documentFields.putAll(in.readMap(StreamInput::readString, DocumentField::new)); + metaFields.putAll(in.readMap(StreamInput::readString, DocumentField::new)); } else { Map fields = readFields(in); - documentFields = new HashMap<>(); - metaFields = new HashMap<>(); fields.forEach( (fieldName, docField) -> (MapperService.isMetadataFieldStatic(fieldName) ? metaFields : documentFields).put( fieldName, @@ -437,10 +427,14 @@ public DocumentField field(String fieldName) { * */ public void setDocumentField(String fieldName, DocumentField field) { if (fieldName == null || field == null) return; - if (documentFields.size() == 0) this.documentFields = new HashMap<>(); this.documentFields.put(fieldName, field); } + public void addDocumentFields(Map docFields, Map metaFields) { + this.documentFields.putAll(docFields); + this.metaFields.putAll(metaFields); + } + /** * @return a map of metadata fields for this hit */ @@ -846,7 +840,8 @@ public static SearchHit createFromMap(Map values) { Map metaFields = get(METADATA_FIELDS, values, Collections.emptyMap()); Map documentFields = get(DOCUMENT_FIELDS, values, Collections.emptyMap()); - SearchHit searchHit = new SearchHit(-1, id, nestedIdentity, documentFields, metaFields); + SearchHit searchHit = new SearchHit(-1, id, nestedIdentity); + searchHit.addDocumentFields(documentFields, metaFields); String index = get(Fields._INDEX, values, null); String clusterAlias = null; if (index != null) { diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index 53a456c55b07b..8d6b7f17020cc 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -216,6 +216,7 @@ import org.elasticsearch.search.fetch.subphase.MatchedQueriesPhase; import org.elasticsearch.search.fetch.subphase.ScriptFieldsPhase; import org.elasticsearch.search.fetch.subphase.SeqNoPrimaryTermPhase; +import org.elasticsearch.search.fetch.subphase.StoredFieldsPhase; import org.elasticsearch.search.fetch.subphase.highlight.FastVectorHighlighter; import org.elasticsearch.search.fetch.subphase.highlight.HighlightPhase; import org.elasticsearch.search.fetch.subphase.highlight.Highlighter; @@ -989,6 +990,7 @@ private void registerSignificanceHeuristic(Sig private void registerFetchSubPhases(List plugins) { registerFetchSubPhase(new ExplainPhase()); + registerFetchSubPhase(new StoredFieldsPhase()); registerFetchSubPhase(new FetchDocValuesPhase()); registerFetchSubPhase(new ScriptFieldsPhase()); registerFetchSubPhase(new FetchSourcePhase()); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java index 0786f2952074e..3655bd58408e8 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java @@ -9,6 +9,7 @@ package org.elasticsearch.search.fetch; import org.apache.lucene.search.Query; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.SearchExecutionContext; @@ -36,16 +37,53 @@ public class FetchContext { private final SearchContext searchContext; - private final SearchLookup searchLookup; private final SourceLoader sourceLoader; + private final FetchSourceContext fetchSourceContext; + private final StoredFieldsContext storedFieldsContext; /** * Create a FetchContext based on a SearchContext */ public FetchContext(SearchContext searchContext) { this.searchContext = searchContext; - this.searchLookup = searchContext.getSearchExecutionContext().lookup(); this.sourceLoader = searchContext.newSourceLoader(); + this.storedFieldsContext = buildStoredFieldsContext(searchContext); + this.fetchSourceContext = buildFetchSourceContext(searchContext); + } + + private static FetchSourceContext buildFetchSourceContext(SearchContext in) { + FetchSourceContext fsc = in.fetchSourceContext(); + StoredFieldsContext sfc = in.storedFieldsContext(); + if (fsc == null) { + boolean hasStoredFields = in.hasStoredFields(); + boolean hasScriptFields = in.hasScriptFields(); + // TODO it seems a bit odd that we disable implicit source loading if we've asked + // for stored fields or script fields? But not eg doc_value fields or via + // the `fields` API + if (hasStoredFields == false && hasScriptFields == false) { + fsc = FetchSourceContext.of(true); + } + } + if (sfc != null && sfc.fetchFields()) { + for (String field : sfc.fieldNames()) { + if (SourceFieldMapper.NAME.equals(field)) { + fsc = fsc == null ? FetchSourceContext.of(true) : FetchSourceContext.of(true, fsc.includes(), fsc.excludes()); + } + } + } + if (sfc != null && sfc.fetchFields() == false) { + fsc = null; + } + return fsc; + } + + private static StoredFieldsContext buildStoredFieldsContext(SearchContext in) { + StoredFieldsContext sfc = in.storedFieldsContext(); + if (sfc == null) { + // if nothing is requested then we just do a standard metadata stored fields request + sfc = StoredFieldsContext.metadataOnly(); + } + return sfc; } /** @@ -66,7 +104,7 @@ public ContextIndexSearcher searcher() { * The {@code SearchLookup} for the this context */ public SearchLookup searchLookup() { - return searchLookup; + return searchContext.getSearchExecutionContext().lookup(); } /** @@ -101,7 +139,14 @@ public ParsedQuery parsedPostFilter() { * Configuration for fetching _source */ public FetchSourceContext fetchSourceContext() { - return searchContext.fetchSourceContext(); + return this.fetchSourceContext; + } + + /** + * Configuration for fetching stored fields + */ + public StoredFieldsContext storedFieldsContext() { + return storedFieldsContext; } /** diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index 5f6ea8b32f0c5..5540a11eb2c1e 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -12,16 +12,12 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.TotalHits; -import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.SourceLoader; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.LeafNestedDocuments; import org.elasticsearch.search.NestedDocuments; import org.elasticsearch.search.SearchContextSourcePrinter; @@ -29,7 +25,6 @@ import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.fetch.FetchSubPhase.HitContext; -import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.InnerHitsContext; import org.elasticsearch.search.fetch.subphase.InnerHitsPhase; import org.elasticsearch.search.internal.SearchContext; @@ -43,18 +38,11 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.function.Function; import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static java.util.Collections.emptyMap; /** * Fetch phase of a search request, used to fetch the actual top matching documents to be returned to the client, identified @@ -102,15 +90,20 @@ public void execute(SearchContext context) { private SearchHits buildSearchHits(SearchContext context, Profiler profiler) { - SourceLoader sourceLoader = context.newSourceLoader(); - Map> storedToRequestedFields = new HashMap<>(); - StoredFieldLoader storedFieldLoader = profiler.storedFields( - createStoredFieldLoader(context, sourceLoader, storedToRequestedFields) - ); - FetchContext fetchContext = new FetchContext(context); + SourceLoader sourceLoader = context.newSourceLoader(); List processors = getProcessors(context.shardTarget(), fetchContext, profiler); + + StoredFieldsSpec storedFieldsSpec = StoredFieldsSpec.NO_REQUIREMENTS; + for (FetchSubPhaseProcessor proc : processors) { + storedFieldsSpec = storedFieldsSpec.merge(proc.storedFieldsSpec()); + } + storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(false, false, sourceLoader.requiredStoredFields())); + + StoredFieldLoader storedFieldLoader = profiler.storedFields(buildStoredFieldsLoader(storedFieldsSpec)); + boolean requiresSource = storedFieldsSpec.requiresSource(); + NestedDocuments nestedDocuments = context.getSearchExecutionContext().getNestedDocuments(); FetchPhaseDocsIterator docsIterator = new FetchPhaseDocsIterator() { @@ -126,7 +119,7 @@ protected void setNextReader(LeafReaderContext ctx, int[] docsInLeaf) throws IOE this.ctx = ctx; this.leafNestedDocuments = nestedDocuments.getLeafNestedDocuments(ctx); this.leafStoredFieldLoader = storedFieldLoader.getLoader(ctx, docsInLeaf); - this.leafSourceLoader = fetchContext.sourceLoader().leaf(ctx.reader(), docsInLeaf); + this.leafSourceLoader = sourceLoader.leaf(ctx.reader(), docsInLeaf); for (FetchSubPhaseProcessor processor : processors) { processor.setNextReader(ctx); } @@ -140,11 +133,11 @@ protected SearchHit nextDoc(int doc) throws IOException { } HitContext hit = prepareHitContext( context, + requiresSource, profiler, leafNestedDocuments, leafStoredFieldLoader, doc, - storedToRequestedFields, ctx, leafSourceLoader ); @@ -165,6 +158,13 @@ protected SearchHit nextDoc(int doc) throws IOException { return new SearchHits(hits, totalHits, context.queryResult().getMaxScore()); } + private static StoredFieldLoader buildStoredFieldsLoader(StoredFieldsSpec spec) { + if (spec.noRequirements()) { + return StoredFieldLoader.empty(); + } + return StoredFieldLoader.create(spec.requiresSource(), spec.requiredStoredFields()); + } + List getProcessors(SearchShardTarget target, FetchContext context, Profiler profiler) { try { List processors = new ArrayList<>(); @@ -180,91 +180,33 @@ List getProcessors(SearchShardTarget target, FetchContex } } - private static StoredFieldLoader createStoredFieldLoader( - SearchContext context, - SourceLoader sourceLoader, - Map> storedToRequestedFields - ) { - StoredFieldsContext storedFieldsContext = context.storedFieldsContext(); - - if (storedFieldsContext == null) { - // no fields specified, default to return source if no explicit indication - if (context.hasScriptFields() == false && context.hasFetchSourceContext() == false) { - context.fetchSourceContext(FetchSourceContext.FETCH_SOURCE); - } - boolean loadSource = sourceRequired(context); - if (loadSource) { - if (false == sourceLoader.requiredStoredFields().isEmpty()) { - // add the stored fields needed to load the source mapping to an empty set so they aren't returned - sourceLoader.requiredStoredFields().forEach(fieldName -> storedToRequestedFields.putIfAbsent(fieldName, Set.of())); - } - } - return StoredFieldLoader.create(loadSource, sourceLoader.requiredStoredFields()); - } else if (storedFieldsContext.fetchFields() == false) { - // disable stored fields entirely - return StoredFieldLoader.empty(); - } else { - for (String fieldNameOrPattern : context.storedFieldsContext().fieldNames()) { - if (fieldNameOrPattern.equals(SourceFieldMapper.NAME)) { - FetchSourceContext fetchSourceContext = context.hasFetchSourceContext() - ? context.fetchSourceContext() - : FetchSourceContext.FETCH_SOURCE; - context.fetchSourceContext(FetchSourceContext.of(true, fetchSourceContext.includes(), fetchSourceContext.excludes())); - continue; - } - SearchExecutionContext searchExecutionContext = context.getSearchExecutionContext(); - Collection fieldNames = searchExecutionContext.getMatchingFieldNames(fieldNameOrPattern); - for (String fieldName : fieldNames) { - MappedFieldType fieldType = searchExecutionContext.getFieldType(fieldName); - String storedField = fieldType.name(); - Set requestedFields = storedToRequestedFields.computeIfAbsent(storedField, key -> new HashSet<>()); - requestedFields.add(fieldName); - } - } - boolean loadSource = sourceRequired(context); - if (loadSource) { - sourceLoader.requiredStoredFields().forEach(fieldName -> storedToRequestedFields.putIfAbsent(fieldName, Set.of())); - } - if (storedToRequestedFields.isEmpty()) { - // empty list specified, default to disable _source if no explicit indication - return StoredFieldLoader.create(loadSource, sourceLoader.requiredStoredFields()); - } else { - return StoredFieldLoader.create(loadSource, storedToRequestedFields.keySet()); - } - } - } - - private static boolean sourceRequired(SearchContext context) { - return context.sourceRequested() || context.fetchFieldsContext() != null; - } - private static HitContext prepareHitContext( SearchContext context, + boolean requiresSource, Profiler profiler, LeafNestedDocuments nestedDocuments, LeafStoredFieldLoader leafStoredFieldLoader, int docId, - Map> storedToRequestedFields, LeafReaderContext subReaderContext, SourceLoader.Leaf sourceLoader ) throws IOException { if (nestedDocuments.advance(docId - subReaderContext.docBase) == null) { return prepareNonNestedHitContext( context, + requiresSource, profiler, leafStoredFieldLoader, docId, - storedToRequestedFields, subReaderContext, sourceLoader ); } else { return prepareNestedHitContext( context, + requiresSource, profiler, docId, nestedDocuments, - storedToRequestedFields, subReaderContext, leafStoredFieldLoader ); @@ -280,10 +222,10 @@ private static HitContext prepareHitContext( */ private static HitContext prepareNonNestedHitContext( SearchContext context, + boolean requiresSource, Profiler profiler, LeafStoredFieldLoader leafStoredFieldLoader, int docId, - Map> storedToRequestedFields, LeafReaderContext subReaderContext, SourceLoader.Leaf sourceLoader ) throws IOException { @@ -292,22 +234,13 @@ private static HitContext prepareNonNestedHitContext( leafStoredFieldLoader.advanceTo(subDocId); if (leafStoredFieldLoader.id() == null) { - SearchHit hit = new SearchHit(docId, null, null, null); + SearchHit hit = new SearchHit(docId, null); Source source = Source.lazy(lazyStoredSourceLoader(profiler, subReaderContext, subDocId)); - return new HitContext(hit, subReaderContext, subDocId, source); + return new HitContext(hit, subReaderContext, subDocId, Map.of(), source); } else { - SearchHit hit; - if (leafStoredFieldLoader.storedFields().isEmpty() == false) { - Map docFields = new HashMap<>(); - Map metaFields = new HashMap<>(); - fillDocAndMetaFields(context, leafStoredFieldLoader.storedFields(), storedToRequestedFields, docFields, metaFields); - hit = new SearchHit(docId, leafStoredFieldLoader.id(), docFields, metaFields); - } else { - hit = new SearchHit(docId, leafStoredFieldLoader.id(), emptyMap(), emptyMap()); - } - + SearchHit hit = new SearchHit(docId, leafStoredFieldLoader.id()); Source source; - if (sourceRequired(context)) { + if (requiresSource) { try { profiler.startLoadingSource(); source = Source.fromBytes(sourceLoader.source(leafStoredFieldLoader, subDocId)); @@ -320,7 +253,7 @@ private static HitContext prepareNonNestedHitContext( } else { source = Source.lazy(lazyStoredSourceLoader(profiler, subReaderContext, subDocId)); } - return new HitContext(hit, subReaderContext, subDocId, source); + return new HitContext(hit, subReaderContext, subDocId, leafStoredFieldLoader.storedFields(), source); } } @@ -348,17 +281,13 @@ private static Supplier lazyStoredSourceLoader(Profiler profiler, LeafRe @SuppressWarnings("unchecked") private static HitContext prepareNestedHitContext( SearchContext context, + boolean requiresSource, Profiler profiler, int topDocId, LeafNestedDocuments nestedInfo, - Map> storedToRequestedFields, LeafReaderContext subReaderContext, LeafStoredFieldLoader childFieldLoader ) throws IOException { - // Also if highlighting is requested on nested documents we need to fetch the _source from the root document, - // otherwise highlighting will attempt to fetch the _source from the nested doc, which will fail, - // because the entire _source is only stored with the root document. - boolean needSource = sourceRequired(context) || context.highlight() != null; String rootId; Map rootSourceAsMap = null; @@ -367,18 +296,18 @@ private static HitContext prepareNestedHitContext( if (context instanceof InnerHitsContext.InnerHitSubContext innerHitsContext) { rootId = innerHitsContext.getRootId(); - if (needSource) { + if (requiresSource) { Source rootLookup = innerHitsContext.getRootLookup(); rootSourceAsMap = rootLookup.source(); rootSourceContentType = rootLookup.sourceContentType(); } } else { - StoredFieldLoader rootLoader = profiler.storedFields(StoredFieldLoader.create(needSource, Collections.emptySet())); + StoredFieldLoader rootLoader = profiler.storedFields(StoredFieldLoader.create(requiresSource, Collections.emptySet())); LeafStoredFieldLoader leafRootLoader = rootLoader.getLoader(subReaderContext, null); leafRootLoader.advanceTo(nestedInfo.rootDoc()); rootId = leafRootLoader.id(); - if (needSource) { + if (requiresSource) { if (leafRootLoader.source() != null) { Tuple> tuple = XContentHelper.convertToMap(leafRootLoader.source(), false); rootSourceAsMap = tuple.v2(); @@ -389,20 +318,11 @@ private static HitContext prepareNestedHitContext( } } - Map docFields = emptyMap(); - Map metaFields = emptyMap(); - if (context.hasStoredFields() && context.storedFieldsContext().fieldNames().isEmpty() == false) { - childFieldLoader.advanceTo(nestedInfo.doc()); - if (childFieldLoader.storedFields().isEmpty() == false) { - docFields = new HashMap<>(); - metaFields = new HashMap<>(); - fillDocAndMetaFields(context, childFieldLoader.storedFields(), storedToRequestedFields, docFields, metaFields); - } - } + childFieldLoader.advanceTo(nestedInfo.doc()); SearchHit.NestedIdentity nestedIdentity = nestedInfo.nestedIdentity(); - SearchHit hit = new SearchHit(topDocId, rootId, nestedIdentity, docFields, metaFields); + SearchHit hit = new SearchHit(topDocId, rootId, nestedIdentity); if (rootSourceAsMap != null && rootSourceAsMap.isEmpty() == false) { // Isolate the nested json array object that matches with nested hit and wrap it back into the same json @@ -427,47 +347,15 @@ private static HitContext prepareNestedHitContext( current = next; } } - return new HitContext(hit, subReaderContext, nestedInfo.doc(), Source.fromMap(nestedSourceAsMap, rootSourceContentType)); - } - return new HitContext(hit, subReaderContext, nestedInfo.doc(), Source.empty(rootSourceContentType)); - } - - public static List processStoredField(Function fieldTypeLookup, String field, List input) { - MappedFieldType ft = fieldTypeLookup.apply(field); - if (ft == null) { - // TODO remove this once we've stopped calling it for accidentally loaded fields - return input; - } - return input.stream().map(ft::valueForDisplay).collect(Collectors.toList()); - } - - private static void fillDocAndMetaFields( - SearchContext context, - Map> storedFields, - Map> storedToRequestedFields, - Map docFields, - Map metaFields - ) { - Function fieldTypeLookup = context.getSearchExecutionContext()::getFieldType; - for (Map.Entry> entry : storedFields.entrySet()) { - String storedField = entry.getKey(); - List storedValues = processStoredField(fieldTypeLookup, storedField, entry.getValue()); - if (storedToRequestedFields.containsKey(storedField)) { - for (String requestedField : storedToRequestedFields.get(storedField)) { - if (context.getSearchExecutionContext().isMetadataField(requestedField)) { - metaFields.put(requestedField, new DocumentField(requestedField, storedValues)); - } else { - docFields.put(requestedField, new DocumentField(requestedField, storedValues)); - } - } - } else { - if (context.getSearchExecutionContext().isMetadataField(storedField)) { - metaFields.put(storedField, new DocumentField(storedField, storedValues)); - } else { - docFields.put(storedField, new DocumentField(storedField, storedValues)); - } - } + return new HitContext( + hit, + subReaderContext, + nestedInfo.doc(), + childFieldLoader.storedFields(), + Source.fromMap(nestedSourceAsMap, rootSourceContentType) + ); } + return new HitContext(hit, subReaderContext, nestedInfo.doc(), childFieldLoader.storedFields(), Source.empty(null)); } interface Profiler { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchProfiler.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchProfiler.java index 728b1ba6a2372..3e3d654c0a393 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchProfiler.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchProfiler.java @@ -122,6 +122,11 @@ public void setNextReader(LeafReaderContext readerContext) throws IOException { } } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return delegate.storedFieldsSpec(); + } + @Override public void process(HitContext hitContext) throws IOException { Timer timer = breakdown.getTimer(FetchSubPhaseTiming.PROCESS); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java index 4d22a601b980f..a6f41f8b7fed3 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java @@ -15,6 +15,8 @@ import org.elasticsearch.search.lookup.Source; import java.io.IOException; +import java.util.List; +import java.util.Map; /** * Sub phase within the fetch phase used to fetch things *about* the documents like highlighting or matched queries. @@ -26,12 +28,14 @@ class HitContext { private final LeafReaderContext readerContext; private final int docId; private final Source source; + private final Map> loadedFields; - public HitContext(SearchHit hit, LeafReaderContext context, int docId, Source source) { + public HitContext(SearchHit hit, LeafReaderContext context, int docId, Map> loadedFields, Source source) { this.hit = hit; this.readerContext = context; this.docId = docId; this.source = source; + this.loadedFields = loadedFields; } public SearchHit hit() { @@ -64,6 +68,10 @@ public Source source() { return source; } + public Map> loadedFields() { + return loadedFields; + } + public IndexReader topLevelReader() { return ReaderUtil.getTopLevelContext(readerContext).reader(); } @@ -76,4 +84,5 @@ public IndexReader topLevelReader() { * implementation should return {@code null} */ FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) throws IOException; + } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseProcessor.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseProcessor.java index b2a1058e62c35..316c61e4db1dc 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseProcessor.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseProcessor.java @@ -36,4 +36,9 @@ public interface FetchSubPhaseProcessor { default Map getDebugInfo() { return null; } + + /** + * The stored fields or source required by this sub phase + */ + StoredFieldsSpec storedFieldsSpec(); } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsContext.java b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsContext.java index a04904d24ccb3..ae0e52ab69091 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsContext.java @@ -31,7 +31,7 @@ public class StoredFieldsContext implements Writeable { public static final String _NONE_ = "_none_"; private final List fieldNames; - private boolean fetchFields; + private final boolean fetchFields; private StoredFieldsContext(boolean fetchFields) { this.fetchFields = fetchFields; @@ -143,6 +143,10 @@ public void toXContent(String preferredName, XContentBuilder builder) throws IOE } } + public static StoredFieldsContext metadataOnly() { + return new StoredFieldsContext(true); + } + public static StoredFieldsContext fromList(List fieldNames) { if (fieldNames.size() == 1 && _NONE_.equals(fieldNames.get(0))) { return new StoredFieldsContext(false); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java new file mode 100644 index 0000000000000..0b68de0fc0242 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.fetch; + +import java.util.HashSet; +import java.util.Set; + +/** + * Defines which stored fields need to be loaded during a fetch + * @param requiresSource should source be loaded + * @param requiredStoredFields a set of stored fields to load + */ +public record StoredFieldsSpec(boolean requiresSource, boolean requiresMetadata, Set requiredStoredFields) { + + public boolean noRequirements() { + return requiresSource == false && requiresMetadata == false && requiredStoredFields.isEmpty(); + } + + /** + * Use when no stored fields are required + */ + public static StoredFieldsSpec NO_REQUIREMENTS = new StoredFieldsSpec(false, false, Set.of()); + + /** + * Use when the source should be loaded but no other stored fields are required + */ + public static StoredFieldsSpec NEEDS_SOURCE = new StoredFieldsSpec(true, false, Set.of()); + + /** + * Combine these stored field requirements with those from another StoredFieldsSpec + */ + public StoredFieldsSpec merge(StoredFieldsSpec other) { + Set mergedFields = new HashSet<>(this.requiredStoredFields); + mergedFields.addAll(other.requiredStoredFields); + return new StoredFieldsSpec( + this.requiresSource || other.requiresSource, + this.requiresMetadata || other.requiresMetadata, + mergedFields + ); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java index a28b4fef89c88..0873ca777d428 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java @@ -12,6 +12,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.rescore.RescoreContext; import java.io.IOException; @@ -42,6 +43,11 @@ public void process(HitContext hitContext) throws IOException { // we use the top level doc id, since we work with the top level searcher hitContext.hit().explanation(explanation); } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } }; } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java index 3a5df16856312..30177c780f4a3 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java @@ -15,6 +15,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.ArrayList; @@ -59,6 +60,11 @@ public void setNextReader(LeafReaderContext readerContext) { } } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + @Override public void process(HitContext hit) throws IOException { for (DocValueField f : fields) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java index 501aa0b1979dc..c0bfad44195cd 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.Map; @@ -39,6 +40,12 @@ public void setNextReader(LeafReaderContext readerContext) { fieldFetcher.setNextReader(readerContext); } + @Override + public StoredFieldsSpec storedFieldsSpec() { + // TODO can we get finer-grained information from the FieldFetcher for this? + return StoredFieldsSpec.NEEDS_SOURCE; + } + @Override public void process(HitContext hitContext) throws IOException { Map documentFields = fieldFetcher.fetch(hitContext.source(), hitContext.docId()); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchScorePhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchScorePhase.java index 906ae69552c5f..897c94da3a9b3 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchScorePhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchScorePhase.java @@ -17,6 +17,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; @@ -41,6 +42,11 @@ public void setNextReader(LeafReaderContext readerContext) throws IOException { scorer = scorerSupplier.get(1L); // random-access } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + @Override public void process(HitContext hitContext) throws IOException { if (scorer == null || scorer.iterator().advance(hitContext.docId()) != hitContext.docId()) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java index 575555f6f4dcd..3b8e4e69d9318 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java @@ -13,6 +13,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.search.lookup.SourceFilter; @@ -25,7 +26,6 @@ public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) { if (fetchSourceContext == null || fetchSourceContext.fetchSource() == false) { return null; } - String index = fetchContext.getIndexName(); assert fetchSourceContext.fetchSource(); SourceFilter sourceFilter = fetchSourceContext.filter(); @@ -37,8 +37,14 @@ public void setNextReader(LeafReaderContext readerContext) { } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NEEDS_SOURCE; + } + @Override public void process(HitContext hitContext) { + String index = fetchContext.getIndexName(); if (fetchContext.getSearchExecutionContext().isSourceEnabled() == false) { if (fetchSourceContext.hasFilter()) { throw new IllegalArgumentException( diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchVersionPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchVersionPhase.java index 8b89aecc4761e..67cf97894e079 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchVersionPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchVersionPhase.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; @@ -32,6 +33,11 @@ public void setNextReader(LeafReaderContext readerContext) throws IOException { versions = readerContext.reader().getNumericDocValues(VersionFieldMapper.NAME); } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + @Override public void process(HitContext hitContext) throws IOException { long version = Versions.NOT_FOUND; diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java index 8e062e47ca409..44e9a2a6e5193 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java @@ -19,11 +19,15 @@ import org.elasticsearch.search.fetch.FetchSearchResult; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; +import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.lookup.Source; import java.io.IOException; +import java.util.Collection; import java.util.HashMap; import java.util.Map; +import java.util.Set; public final class InnerHitsPhase implements FetchSubPhase { @@ -39,12 +43,18 @@ public FetchSubPhaseProcessor getProcessor(FetchContext searchContext) { return null; } Map innerHits = searchContext.innerHits().getInnerHits(); + StoredFieldsSpec storedFieldsSpec = new StoredFieldsSpec(requiresSource(innerHits.values()), false, Set.of()); return new FetchSubPhaseProcessor() { @Override public void setNextReader(LeafReaderContext readerContext) { } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return storedFieldsSpec; + } + @Override public void process(HitContext hitContext) throws IOException { SearchHit hit = hitContext.hit(); @@ -54,6 +64,16 @@ public void process(HitContext hitContext) throws IOException { }; } + private static boolean requiresSource(Collection subContexts) { + boolean requiresSource = false; + for (SearchContext sc : subContexts) { + requiresSource |= sc.sourceRequested(); + requiresSource |= sc.fetchFieldsContext() != null; + requiresSource |= sc.highlight() != null; + } + return requiresSource; + } + private void hitExecute(Map innerHits, SearchHit hit, Source rootSource) throws IOException { for (Map.Entry entry : innerHits.entrySet()) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java index 6ff1e698d96af..883bae9443bf3 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java @@ -17,6 +17,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.ArrayList; @@ -71,6 +72,11 @@ public void process(HitContext hitContext) { } hitContext.hit().matchedQueries(matches.toArray(String[]::new)); } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } }; } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsContext.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsContext.java index bcdfb2ed288cb..0725e6cb32fa6 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsContext.java @@ -39,7 +39,7 @@ public boolean ignoreException() { } } - private List fields = new ArrayList<>(); + private final List fields = new ArrayList<>(); public ScriptFieldsContext() {} diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsPhase.java index 1a84051a285dc..965ef59adb4f2 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsPhase.java @@ -14,12 +14,14 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Set; public final class ScriptFieldsPhase implements FetchSubPhase { @Override @@ -37,6 +39,15 @@ public void setNextReader(LeafReaderContext readerContext) { leafScripts = createLeafScripts(readerContext, scriptFields); } + @Override + public StoredFieldsSpec storedFieldsSpec() { + // If script fields need source then they will load it via SearchLookup, + // which has its own lazy loading config that kicks in if not overridden + // by other sub phases that require source. However, if script fields + // are present then we enforce metadata loading + return new StoredFieldsSpec(false, true, Set.of()); + } + @Override public void process(HitContext hitContext) { int docId = hitContext.docId(); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermPhase.java index ae606a87f60f5..13e26544af20a 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermPhase.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; @@ -34,6 +35,11 @@ public void setNextReader(LeafReaderContext readerContext) throws IOException { primaryTermField = readerContext.reader().getNumericDocValues(SeqNoFieldMapper.PRIMARY_TERM_NAME); } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + @Override public void process(HitContext hitContext) throws IOException { int docId = hitContext.docId(); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/StoredFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/StoredFieldsPhase.java new file mode 100644 index 0000000000000..d6950df962433 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/StoredFieldsPhase.java @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.fetch.subphase; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.index.mapper.IgnoredFieldMapper; +import org.elasticsearch.index.mapper.LegacyTypeFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.RoutingFieldMapper; +import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.fetch.FetchContext; +import org.elasticsearch.search.fetch.FetchSubPhase; +import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsContext; +import org.elasticsearch.search.fetch.StoredFieldsSpec; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Process stored fields loaded from a HitContext into DocumentFields + */ +public class StoredFieldsPhase implements FetchSubPhase { + + /** Associates a field name with a mapped field type and whether or not it is a metadata field */ + private record StoredField(String name, MappedFieldType ft, boolean isMetadataField) { + + /** Processes a set of stored fields using field type information */ + List process(Map> loadedFields) { + List inputs = loadedFields.get(ft.name()); + if (inputs == null) { + return List.of(); + } + return inputs.stream().map(ft::valueForDisplay).toList(); + } + + boolean hasValue(Map> loadedFields) { + return loadedFields.containsKey(ft.name()); + } + + } + + private static final List METADATA_FIELDS = List.of( + new StoredField("_routing", RoutingFieldMapper.FIELD_TYPE, true), + new StoredField("_ignored", IgnoredFieldMapper.FIELD_TYPE, true), + // pre-6.0 indexes can return a _type field, this will be valueless in modern indexes and ignored + new StoredField("_type", LegacyTypeFieldMapper.FIELD_TYPE, true) + ); + + @Override + public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) { + StoredFieldsContext storedFieldsContext = fetchContext.storedFieldsContext(); + if (storedFieldsContext == null || storedFieldsContext.fetchFields() == false) { + return null; + } + + // build the StoredFieldsSpec and a list of StoredField records to process + List storedFields = new ArrayList<>(METADATA_FIELDS); + Set fieldsToLoad = new HashSet<>(); + if (storedFieldsContext.fieldNames() != null) { + SearchExecutionContext sec = fetchContext.getSearchExecutionContext(); + for (String field : storedFieldsContext.fieldNames()) { + if (SourceFieldMapper.NAME.equals(field) == false) { + Collection fieldNames = sec.getMatchingFieldNames(field); + for (String fieldName : fieldNames) { + MappedFieldType ft = sec.getFieldType(fieldName); + if (ft.isStored() == false) { + continue; + } + storedFields.add(new StoredField(fieldName, ft, sec.isMetadataField(ft.name()))); + fieldsToLoad.add(ft.name()); + } + } + } + } + StoredFieldsSpec storedFieldsSpec = new StoredFieldsSpec(false, true, fieldsToLoad); + + return new FetchSubPhaseProcessor() { + @Override + public void setNextReader(LeafReaderContext readerContext) { + + } + + @Override + public void process(HitContext hitContext) { + Map> loadedFields = hitContext.loadedFields(); + Map docFields = new HashMap<>(); + Map metaFields = new HashMap<>(); + for (StoredField storedField : storedFields) { + if (storedField.hasValue(loadedFields)) { + DocumentField df = new DocumentField(storedField.name, storedField.process(loadedFields)); + if (storedField.isMetadataField) { + metaFields.put(storedField.name, df); + } else { + docFields.put(storedField.name, df); + } + } + } + hitContext.hit().addDocumentFields(docFields, metaFields); + } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return storedFieldsSpec; + } + }; + } + +} diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java index 4301e8734fcf9..4b60d32d3065f 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java @@ -16,12 +16,15 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import java.util.function.Function; public class HighlightPhase implements FetchSubPhase { @@ -43,12 +46,7 @@ public FetchSubPhaseProcessor getProcessor(FetchContext context) { public FetchSubPhaseProcessor getProcessor(FetchContext context, SearchHighlightContext highlightContext, Query query) { Map sharedCache = new HashMap<>(); - Map> contextBuilders = contextBuilders( - context, - highlightContext, - query, - sharedCache - ); + FieldContext fieldContext = contextBuilders(context, highlightContext, query, sharedCache); return new FetchSubPhaseProcessor() { @Override @@ -56,9 +54,15 @@ public void setNextReader(LeafReaderContext readerContext) { } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return fieldContext.storedFieldsSpec; + } + @Override public void process(HitContext hitContext) throws IOException { Map highlightFields = new HashMap<>(); + Map> contextBuilders = fieldContext.builders; for (String field : contextBuilders.keySet()) { FieldHighlightContext fieldContext = contextBuilders.get(field).apply(hitContext); Highlighter highlighter = getHighlighter(fieldContext.field); @@ -87,13 +91,16 @@ private Highlighter getHighlighter(SearchHighlightContext.Field field) { return highlighter; } - private Map> contextBuilders( + private record FieldContext(StoredFieldsSpec storedFieldsSpec, Map> builders) {} + + private FieldContext contextBuilders( FetchContext context, SearchHighlightContext highlightContext, Query query, Map sharedCache ) { Map> builders = new LinkedHashMap<>(); + StoredFieldsSpec storedFieldsSpec = StoredFieldsSpec.NO_REQUIREMENTS; for (SearchHighlightContext.Field field : highlightContext.fields()) { Highlighter highlighter = getHighlighter(field); @@ -104,8 +111,11 @@ private Map> contextBuilders throw new IllegalArgumentException("source is forced for fields " + fieldNamesToHighlight + " but _source is disabled"); } } + boolean forceSource = highlightContext.forceSource(field); boolean fieldNameContainsWildcards = field.field().contains("*"); + Set storedFields = new HashSet<>(); + boolean sourceRequired = forceSource; for (String fieldName : fieldNamesToHighlight) { MappedFieldType fieldType = context.getSearchExecutionContext().getFieldType(fieldName); @@ -129,9 +139,14 @@ private Map> contextBuilders } } + if (fieldType.isStored()) { + storedFields.add(fieldType.name()); + } else { + sourceRequired = true; + } + Query highlightQuery = field.fieldOptions().highlightQuery(); - boolean forceSource = highlightContext.forceSource(field); builders.put( fieldName, hc -> new FieldHighlightContext( @@ -146,7 +161,11 @@ private Map> contextBuilders ) ); } + // TODO in future we can load the storedFields in advance here and make use of them, + // but for now they are loaded separately in HighlightUtils so we only return whether + // or not we need source. + storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(sourceRequired, false, Set.of())); } - return builders; + return new FieldContext(storedFieldsSpec, builders); } } diff --git a/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java index 638dfec4927e9..79e762be7a61a 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java @@ -90,7 +90,7 @@ public void suggest(SuggestionSearchContext suggest) { @Override public boolean hasScriptFields() { - return scriptFields != null; + return scriptFields != null && scriptFields.fields().isEmpty() == false; } @Override diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index 01054e2b74230..0f2c55b586b7c 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -44,9 +44,7 @@ public void testCollapseSingleHit() throws IOException { List collapsedHits = new ArrayList<>(numInnerHits); for (int innerHitNum = 0; innerHitNum < numInnerHits; innerHitNum++) { SearchHits hits = new SearchHits( - new SearchHit[] { - new SearchHit(innerHitNum, "ID", Collections.emptyMap(), Collections.emptyMap()), - new SearchHit(innerHitNum + 1, "ID", Collections.emptyMap(), Collections.emptyMap()) }, + new SearchHit[] { new SearchHit(innerHitNum, "ID"), new SearchHit(innerHitNum + 1, "ID") }, new TotalHits(2, TotalHits.Relation.EQUAL_TO), 1.0F ); @@ -110,17 +108,9 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL } }; - SearchHits hits = new SearchHits( - new SearchHit[] { - new SearchHit( - 1, - "ID", - Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(collapseValue))), - Collections.emptyMap() - ) }, - new TotalHits(1, TotalHits.Relation.EQUAL_TO), - 1.0F - ); + SearchHit hit = new SearchHit(1, "ID"); + hit.setDocumentField("someField", new DocumentField("someField", Collections.singletonList(collapseValue))); + SearchHits hits = new SearchHits(new SearchHit[] { hit }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F); InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1); ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, internalSearchResponse, () -> new SearchPhase("test") { @Override @@ -147,9 +137,7 @@ public void testFailOneItemFailsEntirePhase() throws IOException { AtomicBoolean executedMultiSearch = new AtomicBoolean(false); SearchHits collapsedHits = new SearchHits( - new SearchHit[] { - new SearchHit(2, "ID", Collections.emptyMap(), Collections.emptyMap()), - new SearchHit(3, "ID", Collections.emptyMap(), Collections.emptyMap()) }, + new SearchHit[] { new SearchHit(2, "ID"), new SearchHit(3, "ID") }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F ); @@ -187,23 +175,11 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL } }; - SearchHits hits = new SearchHits( - new SearchHit[] { - new SearchHit( - 1, - "ID", - Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(collapseValue))), - Collections.emptyMap() - ), - new SearchHit( - 2, - "ID2", - Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(collapseValue))), - Collections.emptyMap() - ) }, - new TotalHits(1, TotalHits.Relation.EQUAL_TO), - 1.0F - ); + SearchHit hit1 = new SearchHit(1, "ID"); + hit1.setDocumentField("someField", new DocumentField("someField", Collections.singletonList(collapseValue))); + SearchHit hit2 = new SearchHit(2, "ID2"); + hit2.setDocumentField("someField", new DocumentField("someField", Collections.singletonList(collapseValue))); + SearchHits hits = new SearchHits(new SearchHit[] { hit1, hit2 }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F); InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1); ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, internalSearchResponse, () -> new SearchPhase("test") { @Override @@ -227,23 +203,11 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL } }; - SearchHits hits = new SearchHits( - new SearchHit[] { - new SearchHit( - 1, - "ID", - Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(null))), - Collections.emptyMap() - ), - new SearchHit( - 2, - "ID2", - Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(null))), - Collections.emptyMap() - ) }, - new TotalHits(1, TotalHits.Relation.EQUAL_TO), - 1.0F - ); + SearchHit hit1 = new SearchHit(1, "ID"); + hit1.setDocumentField("someField", new DocumentField("someField", Collections.singletonList(null))); + SearchHit hit2 = new SearchHit(2, "ID2"); + hit2.setDocumentField("someField", new DocumentField("someField", Collections.singletonList(null))); + SearchHits hits = new SearchHits(new SearchHit[] { hit1, hit2 }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F); InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1); ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, internalSearchResponse, () -> new SearchPhase("test") { @Override diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java index 5bfe68fe9637b..79fee3ac34ec1 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java @@ -422,7 +422,7 @@ private static AtomicArray generateFetchResults( List searchHits = new ArrayList<>(); for (ScoreDoc scoreDoc : mergedSearchDocs) { if (scoreDoc.shardIndex == shardIndex) { - searchHits.add(new SearchHit(scoreDoc.doc, "", Collections.emptyMap(), Collections.emptyMap())); + searchHits.add(new SearchHit(scoreDoc.doc, "")); if (scoreDoc.score > maxScore) { maxScore = scoreDoc.score; } @@ -433,7 +433,7 @@ private static AtomicArray generateFetchResults( for (CompletionSuggestion.Entry.Option option : ((CompletionSuggestion) suggestion).getOptions()) { ScoreDoc doc = option.getDoc(); if (doc.shardIndex == shardIndex) { - searchHits.add(new SearchHit(doc.doc, "", Collections.emptyMap(), Collections.emptyMap())); + searchHits.add(new SearchHit(doc.doc, "")); if (doc.score > maxScore) { maxScore = doc.score; } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java index 4815030e73a7b..cd45ddd680332 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java @@ -40,7 +40,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import static java.util.Collections.emptyList; @@ -216,7 +215,7 @@ public void testFromXContentWithFailures() throws IOException { } public void testToXContent() throws IOException { - SearchHit hit = new SearchHit(1, "id1", Collections.emptyMap(), Collections.emptyMap()); + SearchHit hit = new SearchHit(1, "id1"); hit.score(2.0f); SearchHit[] hits = new SearchHit[] { hit }; { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java index 3052bfdb3400d..52475e7b059b5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java @@ -19,7 +19,7 @@ public class IgnoredFieldTypeTests extends FieldTypeTestCase { public void testPrefixQuery() { - MappedFieldType ft = IgnoredFieldMapper.IgnoredFieldType.INSTANCE; + MappedFieldType ft = IgnoredFieldMapper.FIELD_TYPE; Query expected = new PrefixQuery(new Term("_ignored", new BytesRef("foo*"))); assertEquals(expected, ft.prefixQuery("foo*", null, MOCK_CONTEXT)); @@ -36,7 +36,7 @@ public void testPrefixQuery() { } public void testRegexpQuery() { - MappedFieldType ft = IgnoredFieldMapper.IgnoredFieldType.INSTANCE; + MappedFieldType ft = IgnoredFieldMapper.FIELD_TYPE; Query expected = new RegexpQuery(new Term("_ignored", new BytesRef("foo?"))); assertEquals(expected, ft.regexpQuery("foo?", 0, 0, 10, null, MOCK_CONTEXT)); @@ -49,7 +49,7 @@ public void testRegexpQuery() { } public void testWildcardQuery() { - MappedFieldType ft = IgnoredFieldMapper.IgnoredFieldType.INSTANCE; + MappedFieldType ft = IgnoredFieldMapper.FIELD_TYPE; Query expected = new WildcardQuery(new Term("_ignored", new BytesRef("foo*"))); assertEquals(expected, ft.wildcardQuery("foo*", null, MOCK_CONTEXT)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldTypeTests.java index 0d7659418d38e..234e5ff62d111 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldTypeTests.java @@ -18,7 +18,7 @@ public class RoutingFieldTypeTests extends FieldTypeTestCase { public void testPrefixQuery() { - MappedFieldType ft = RoutingFieldMapper.RoutingFieldType.INSTANCE; + MappedFieldType ft = RoutingFieldMapper.FIELD_TYPE; Query expected = new PrefixQuery(new Term("_routing", new BytesRef("foo*"))); assertEquals(expected, ft.prefixQuery("foo*", null, MOCK_CONTEXT)); @@ -35,7 +35,7 @@ public void testPrefixQuery() { } public void testRegexpQuery() { - MappedFieldType ft = RoutingFieldMapper.RoutingFieldType.INSTANCE; + MappedFieldType ft = RoutingFieldMapper.FIELD_TYPE; Query expected = new RegexpQuery(new Term("_routing", new BytesRef("foo?"))); assertEquals(expected, ft.regexpQuery("foo?", 0, 0, 10, null, MOCK_CONTEXT)); @@ -48,7 +48,7 @@ public void testRegexpQuery() { } public void testWildcardQuery() { - MappedFieldType ft = RoutingFieldMapper.RoutingFieldType.INSTANCE; + MappedFieldType ft = RoutingFieldMapper.FIELD_TYPE; Query expected = new WildcardQuery(new Term("_routing", new BytesRef("foo*"))); assertEquals(expected, ft.wildcardQuery("foo*", null, MOCK_CONTEXT)); diff --git a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java index 544c0794271a5..c50e0777bd6de 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java @@ -71,8 +71,8 @@ public static SearchHit createTestItem(XContentType xContentType, boolean withOp documentFields = GetResultTests.randomDocumentFields(xContentType, false).v2(); } } - - SearchHit hit = new SearchHit(internalId, uid, nestedIdentity, documentFields, metaFields); + SearchHit hit = new SearchHit(internalId, uid, nestedIdentity); + hit.addDocumentFields(documentFields, metaFields); if (frequently()) { if (rarely()) { hit.score(Float.NaN); @@ -212,7 +212,7 @@ public void testFromXContentWithoutTypeAndId() throws IOException { } public void testToXContent() throws IOException { - SearchHit searchHit = new SearchHit(1, "id1", Collections.emptyMap(), Collections.emptyMap()); + SearchHit searchHit = new SearchHit(1, "id1"); searchHit.score(1.5f); XContentBuilder builder = JsonXContent.contentBuilder(); searchHit.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -225,25 +225,25 @@ public void testSerializeShardTarget() throws Exception { SearchShardTarget target = new SearchShardTarget("_node_id", new ShardId(new Index("_index", "_na_"), 0), clusterAlias); Map innerHits = new HashMap<>(); - SearchHit innerHit1 = new SearchHit(0, "_id", null, null); + SearchHit innerHit1 = new SearchHit(0, "_id"); innerHit1.shard(target); - SearchHit innerInnerHit2 = new SearchHit(0, "_id", null, null); + SearchHit innerInnerHit2 = new SearchHit(0, "_id"); innerInnerHit2.shard(target); innerHits.put("1", new SearchHits(new SearchHit[] { innerInnerHit2 }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1f)); innerHit1.setInnerHits(innerHits); - SearchHit innerHit2 = new SearchHit(0, "_id", null, null); + SearchHit innerHit2 = new SearchHit(0, "_id"); innerHit2.shard(target); - SearchHit innerHit3 = new SearchHit(0, "_id", null, null); + SearchHit innerHit3 = new SearchHit(0, "_id"); innerHit3.shard(target); innerHits = new HashMap<>(); - SearchHit hit1 = new SearchHit(0, "_id", null, null); + SearchHit hit1 = new SearchHit(0, "_id"); innerHits.put("1", new SearchHits(new SearchHit[] { innerHit1, innerHit2 }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1f)); innerHits.put("2", new SearchHits(new SearchHit[] { innerHit3 }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1f)); hit1.shard(target); hit1.setInnerHits(innerHits); - SearchHit hit2 = new SearchHit(0, "_id", null, null); + SearchHit hit2 = new SearchHit(0, "_id"); hit2.shard(target); SearchHits hits = new SearchHits(new SearchHit[] { hit1, hit2 }, new TotalHits(2, TotalHits.Relation.EQUAL_TO), 1f); @@ -270,7 +270,7 @@ public void testSerializeShardTarget() throws Exception { } public void testNullSource() { - SearchHit searchHit = new SearchHit(0, "_id", null, null); + SearchHit searchHit = new SearchHit(0, "_id"); assertThat(searchHit.getSourceAsMap(), nullValue()); assertThat(searchHit.getSourceRef(), nullValue()); @@ -359,7 +359,8 @@ public void testToXContentEmptyFields() throws IOException { Map fields = new HashMap<>(); fields.put("foo", new DocumentField("foo", Collections.emptyList())); fields.put("bar", new DocumentField("bar", Collections.emptyList())); - SearchHit hit = new SearchHit(0, "_id", fields, Collections.emptyMap()); + SearchHit hit = new SearchHit(0, "_id"); + hit.addDocumentFields(fields, Map.of()); { BytesReference originalBytes = toShuffledXContent(hit, XContentType.JSON, ToXContent.EMPTY_PARAMS, randomBoolean()); // checks that the fields section is completely omitted in the rendering. @@ -377,7 +378,8 @@ public void testToXContentEmptyFields() throws IOException { fields = new HashMap<>(); fields.put("foo", new DocumentField("foo", Collections.emptyList())); fields.put("bar", new DocumentField("bar", Collections.singletonList("value"))); - hit = new SearchHit(0, "_id", fields, Collections.emptyMap()); + hit = new SearchHit(0, "_id"); + hit.addDocumentFields(fields, Collections.emptyMap()); { BytesReference originalBytes = toShuffledXContent(hit, XContentType.JSON, ToXContent.EMPTY_PARAMS, randomBoolean()); final SearchHit parsed; @@ -393,7 +395,8 @@ public void testToXContentEmptyFields() throws IOException { Map metadata = new HashMap<>(); metadata.put("_routing", new DocumentField("_routing", Collections.emptyList())); - hit = new SearchHit(0, "_id", fields, Collections.emptyMap()); + hit = new SearchHit(0, "_id"); + hit.addDocumentFields(fields, Collections.emptyMap()); { BytesReference originalBytes = toShuffledXContent(hit, XContentType.JSON, ToXContent.EMPTY_PARAMS, randomBoolean()); final SearchHit parsed; diff --git a/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java b/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java index 31f348015069f..528223fc40075 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; -import java.util.Collections; import java.util.function.Predicate; public class SearchHitsTests extends AbstractXContentSerializingTestCase { @@ -227,9 +226,7 @@ protected SearchHits doParseInstance(XContentParser parser) throws IOException { } public void testToXContent() throws IOException { - SearchHit[] hits = new SearchHit[] { - new SearchHit(1, "id1", Collections.emptyMap(), Collections.emptyMap()), - new SearchHit(2, "id2", Collections.emptyMap(), Collections.emptyMap()) }; + SearchHit[] hits = new SearchHit[] { new SearchHit(1, "id1"), new SearchHit(2, "id2") }; long totalHits = 1000; float maxScore = 1.5f; @@ -253,10 +250,7 @@ public void testToXContent() throws IOException { public void testFromXContentWithShards() throws IOException { for (boolean withExplanation : new boolean[] { true, false }) { - final SearchHit[] hits = new SearchHit[] { - new SearchHit(1, "id1", Collections.emptyMap(), Collections.emptyMap()), - new SearchHit(2, "id2", Collections.emptyMap(), Collections.emptyMap()), - new SearchHit(10, "id10", Collections.emptyMap(), Collections.emptyMap()) }; + final SearchHit[] hits = new SearchHit[] { new SearchHit(1, "id1"), new SearchHit(2, "id2"), new SearchHit(10, "id10") }; for (SearchHit hit : hits) { String index = randomAlphaOfLengthBetween(5, 10); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java index 0ef8081110df5..16c36697cfc3f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java @@ -154,7 +154,8 @@ private InternalTopHits createTestInstance( Map searchHitFields = new HashMap<>(); scoreDocs[i] = docBuilder.apply(docId, score); - hits[i] = new SearchHit(docId, Integer.toString(i), searchHitFields, Collections.emptyMap()); + hits[i] = new SearchHit(docId, Integer.toString(i)); + hits[i].addDocumentFields(searchHitFields, Collections.emptyMap()); hits[i].score(score); } int totalHits = between(actualSize, 500000); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java b/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java new file mode 100644 index 0000000000000..315c0673e168c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.fetch; + +import org.elasticsearch.script.Script; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.subphase.FetchSourcePhase; +import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext; +import org.elasticsearch.search.fetch.subphase.ScriptFieldsPhase; +import org.elasticsearch.search.fetch.subphase.StoredFieldsPhase; +import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.hasSize; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class StoredFieldsSpecTests extends ESTestCase { + + public void testDefaults() { + SearchSourceBuilder search = new SearchSourceBuilder(); + // defaults - return source and metadata fields + FetchContext fc = new FetchContext(searchContext(search)); + + FetchSubPhaseProcessor sourceProcessor = new FetchSourcePhase().getProcessor(fc); + assertNotNull(sourceProcessor); + StoredFieldsSpec sourceSpec = sourceProcessor.storedFieldsSpec(); + assertTrue(sourceSpec.requiresSource()); + assertThat(sourceSpec.requiredStoredFields(), hasSize(0)); + + FetchSubPhaseProcessor storedFieldsProcessor = new StoredFieldsPhase().getProcessor(fc); + assertNotNull(storedFieldsProcessor); + StoredFieldsSpec storedFieldsSpec = storedFieldsProcessor.storedFieldsSpec(); + assertFalse(storedFieldsSpec.requiresSource()); + assertThat(storedFieldsSpec.requiredStoredFields(), hasSize(0)); + + StoredFieldsSpec merged = sourceSpec.merge(storedFieldsSpec); + assertTrue(merged.requiresSource()); + assertThat(merged.requiredStoredFields(), hasSize(0)); + } + + public void testStoredFieldsDisabled() { + SearchSourceBuilder search = new SearchSourceBuilder(); + search.storedField("_none_"); + FetchContext fc = new FetchContext(searchContext(search)); + + assertNull(new StoredFieldsPhase().getProcessor(fc)); + assertNull(new FetchSourcePhase().getProcessor(fc)); + } + + public void testScriptFieldsEnableMetadata() { + SearchSourceBuilder search = new SearchSourceBuilder(); + search.scriptField("field", new Script("script")); + FetchContext fc = new FetchContext(searchContext(search)); + + FetchSubPhaseProcessor subPhaseProcessor = new ScriptFieldsPhase().getProcessor(fc); + assertNotNull(subPhaseProcessor); + StoredFieldsSpec spec = subPhaseProcessor.storedFieldsSpec(); + assertFalse(spec.requiresSource()); + assertTrue(spec.requiresMetadata()); + } + + private static SearchContext searchContext(SearchSourceBuilder sourceBuilder) { + SearchContext sc = mock(SearchContext.class); + when(sc.fetchSourceContext()).thenReturn(sourceBuilder.fetchSource()); + when(sc.storedFieldsContext()).thenReturn(sourceBuilder.storedFields()); + ScriptFieldsContext scriptContext = new ScriptFieldsContext(); + if (sourceBuilder.scriptFields() != null) { + for (SearchSourceBuilder.ScriptField scriptField : sourceBuilder.scriptFields()) { + scriptContext.add(new ScriptFieldsContext.ScriptField(scriptField.fieldName(), null, true)); + } + } + when(sc.scriptFields()).thenReturn(scriptContext); + return sc; + } + +} diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhaseTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhaseTests.java index 9ae94f385386f..e0a26fbc67ffd 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhaseTests.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.Set; import static org.mockito.ArgumentMatchers.any; @@ -82,7 +83,7 @@ public void testDocValueFetcher() throws IOException { processor.setNextReader(context); for (int doc = 0; doc < context.reader().maxDoc(); doc++) { SearchHit searchHit = new SearchHit(doc + context.docBase); - processor.process(new FetchSubPhase.HitContext(searchHit, context, doc, Source.empty(null))); + processor.process(new FetchSubPhase.HitContext(searchHit, context, doc, Map.of(), Source.empty(null))); assertNotNull(searchHit.getFields().get("field")); } } diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java index 20c736d9974aa..620706a01c88f 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java @@ -170,13 +170,13 @@ private HitContext hitExecuteMultiple( when(sec.isSourceEnabled()).thenReturn(sourceBuilder != null); when(fetchContext.getSearchExecutionContext()).thenReturn(sec); - final SearchHit searchHit = new SearchHit(1, null, nestedIdentity, null, null); + final SearchHit searchHit = new SearchHit(1, null, nestedIdentity); // We don't need a real index, just a LeafReaderContext which cannot be mocked. MemoryIndex index = new MemoryIndex(); LeafReaderContext leafReaderContext = index.createSearcher().getIndexReader().leaves().get(0); Source source = sourceBuilder == null ? Source.empty(null) : Source.fromBytes(BytesReference.bytes(sourceBuilder)); - HitContext hitContext = new HitContext(searchHit, leafReaderContext, 1, source); + HitContext hitContext = new HitContext(searchHit, leafReaderContext, 1, Map.of(), source); FetchSourcePhase phase = new FetchSourcePhase(); FetchSubPhaseProcessor processor = phase.getProcessor(fetchContext); diff --git a/test/framework/src/main/java/org/elasticsearch/search/fetch/HighlighterTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/fetch/HighlighterTestCase.java index 58288542d737c..11bbdbc9b1013 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/fetch/HighlighterTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/fetch/HighlighterTestCase.java @@ -63,9 +63,10 @@ protected final Map highlight(MapperService mapperServic FetchSubPhaseProcessor processor = highlightPhase.getProcessor(fetchContext(context, search)); Source source = Source.fromBytes(doc.source()); FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext( - new SearchHit(0, "id", null, null), + new SearchHit(0, "id"), ir.leaves().get(0), 0, + Map.of(), source ); processor.process(hitContext); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java index d7dfc3f7354d8..e2157345e025f 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; @@ -122,6 +123,11 @@ public List getFetchSubPhases(FetchPhaseConstructionContext conte @Override public void setNextReader(LeafReaderContext readerContext) {} + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + @Override public void process(FetchSubPhase.HitContext hitContext) { if (hitContext.hit().getId().startsWith("boom")) { diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java index 4e8651619c539..42487229a97c4 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java @@ -269,7 +269,7 @@ protected MultiSearchResponse shardOperation(Request request, ShardId shardId) t } return context.getFieldType(field); }); - final SearchHit hit = new SearchHit(scoreDoc.doc, visitor.id(), Map.of(), Map.of()); + final SearchHit hit = new SearchHit(scoreDoc.doc, visitor.id()); hit.sourceRef(filterSource(fetchSourceContext, visitor.source())); hits[j] = hit; } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java index 612a7f5c8efeb..06e823a949235 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java @@ -73,7 +73,7 @@ public void query(QueryRequest r, ActionListener l) { } long sortValue = implicitTiebreakerValues.get(ordinal); - SearchHit searchHit = new SearchHit(ordinal, String.valueOf(ordinal), null, null); + SearchHit searchHit = new SearchHit(ordinal, String.valueOf(ordinal)); searchHit.sortValues( new SearchSortValues( new Long[] { (long) ordinal, sortValue }, @@ -92,7 +92,7 @@ public void fetchHits(Iterable> refs, ActionListener ref : refs) { List hits = new ArrayList<>(ref.size()); for (HitReference hitRef : ref) { - hits.add(new SearchHit(-1, hitRef.id(), null, null)); + hits.add(new SearchHit(-1, hitRef.id())); } searchHits.add(hits); } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java index 336c4a5a18543..90c85c2637634 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java @@ -188,7 +188,8 @@ static class EventsAsHits { Map documentFields = new HashMap<>(); documentFields.put(KEY_FIELD_NAME, new DocumentField(KEY_FIELD_NAME, Collections.singletonList(value.v1()))); // save the timestamp both as docId (int) and as id (string) - SearchHit searchHit = new SearchHit(entry.getKey(), entry.getKey().toString(), documentFields, null); + SearchHit searchHit = new SearchHit(entry.getKey(), entry.getKey().toString()); + searchHit.addDocumentFields(documentFields, Map.of()); hits.add(searchHit); } } @@ -230,7 +231,7 @@ public void fetchHits(Iterable> refs, ActionListener ref : refs) { List hits = new ArrayList<>(ref.size()); for (HitReference hitRef : ref) { - hits.add(new SearchHit(-1, hitRef.id(), null, null)); + hits.add(new SearchHit(-1, hitRef.id())); } searchHits.add(hits); } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/CriterionOrdinalExtractionTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/CriterionOrdinalExtractionTests.java index b992ffe968fcb..224f2bcdca614 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/CriterionOrdinalExtractionTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/CriterionOrdinalExtractionTests.java @@ -26,7 +26,6 @@ import java.util.function.Supplier; import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.eql.EqlTestUtils.randomSearchLongSortValues; import static org.elasticsearch.xpack.eql.EqlTestUtils.randomSearchSortValues; @@ -152,7 +151,8 @@ private SearchHit searchHit(Object timeValue, Object tiebreakerValue, Supplier fields = new HashMap<>(); fields.put(tsField, new DocumentField(tsField, singletonList(timeValue))); fields.put(tbField, new DocumentField(tsField, singletonList(tiebreakerValue))); - SearchHit searchHit = new SearchHit(randomInt(), randomAlphaOfLength(10), fields, emptyMap()); + SearchHit searchHit = new SearchHit(randomInt(), randomAlphaOfLength(10)); + searchHit.addDocumentFields(fields, Map.of()); searchHit.sortValues(searchSortValues.get()); return searchHit; diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java index 7787f3e6ef171..475d8f23573f5 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java @@ -106,7 +106,7 @@ static class TestQueryClient implements QueryClient { @Override public void query(QueryRequest r, ActionListener l) { int ordinal = r.searchSource().terminateAfter(); - SearchHit searchHit = new SearchHit(ordinal, String.valueOf(ordinal), null, null); + SearchHit searchHit = new SearchHit(ordinal, String.valueOf(ordinal)); searchHit.sortValues( new SearchSortValues(new Long[] { (long) ordinal, 1L }, new DocValueFormat[] { DocValueFormat.RAW, DocValueFormat.RAW }) ); @@ -122,7 +122,7 @@ public void fetchHits(Iterable> refs, ActionListener ref : refs) { List hits = new ArrayList<>(ref.size()); for (HitReference hitRef : ref) { - hits.add(new SearchHit(-1, hitRef.id(), null, null)); + hits.add(new SearchHit(-1, hitRef.id())); } searchHits.add(hits); } @@ -386,7 +386,7 @@ private class SuccessfulESMockClient extends ESMockClient { @Override void handleSearchRequest(ActionListener listener, SearchRequest searchRequest) { int ordinal = searchRequest.source().terminateAfter(); - SearchHit searchHit = new SearchHit(ordinal, String.valueOf(ordinal), null, null); + SearchHit searchHit = new SearchHit(ordinal, String.valueOf(ordinal)); searchHit.sortValues( new SearchSortValues(new Long[] { (long) ordinal, 1L }, new DocValueFormat[] { DocValueFormat.RAW, DocValueFormat.RAW }) ); @@ -433,7 +433,7 @@ void handleSearchRequest(ActionListener void } private SearchHits prepareSearchHits() { - SearchHit hit1 = new SearchHit(0, "1", null, null); + SearchHit hit1 = new SearchHit(0, "1"); hit1.score(1f); hit1.shard(new SearchShardTarget("a", new ShardId("a", "indexUUID", 0), null)); - SearchHit hit2 = new SearchHit(0, "2", null, null); + SearchHit hit2 = new SearchHit(0, "2"); hit2.score(1f); hit2.shard(new SearchShardTarget("a", new ShardId("a", "indexUUID", 0), null)); - SearchHit hit3 = new SearchHit(0, "3*", null, null); + SearchHit hit3 = new SearchHit(0, "3*"); hit3.score(1f); hit3.shard(new SearchShardTarget("a", new ShardId("a", "indexUUID", 0), null)); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteForecastActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteForecastActionTests.java index c6ef6350441c8..e9129c450d56f 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteForecastActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportDeleteForecastActionTests.java @@ -85,6 +85,8 @@ private static SearchHit createForecastStatsHit(ForecastRequestStats.ForecastReq ForecastRequestStats.STATUS.getPreferredName(), new DocumentField(ForecastRequestStats.STATUS.getPreferredName(), Collections.singletonList(status.toString())) ); - return new SearchHit(0, "", documentFields, Collections.emptyMap()); + SearchHit hit = new SearchHit(0, ""); + hit.addDocumentFields(documentFields, Map.of()); + return hit; } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java index 9b6ab20af8f80..fcce9c2a38af2 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java @@ -540,7 +540,8 @@ private SearchResponse createSearchResponse(List timestamps, List fields.put(extractedFields.timeField(), new DocumentField("time", Collections.singletonList(timestamps.get(i)))); fields.put("field_1", new DocumentField("field_1", Collections.singletonList(field1Values.get(i)))); fields.put("field_2", new DocumentField("field_2", Collections.singletonList(field2Values.get(i)))); - SearchHit hit = new SearchHit(randomInt(), null, fields, null); + SearchHit hit = new SearchHit(randomInt(), null); + hit.addDocumentFields(fields, Map.of()); hits.add(hit); } SearchHits searchHits = new SearchHits(hits.toArray(new SearchHit[0]), new TotalHits(hits.size(), TotalHits.Relation.EQUAL_TO), 1); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoinerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoinerTests.java index e53b858fbf218..99dfd9e919a6a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoinerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoinerTests.java @@ -309,7 +309,7 @@ private void givenDataFrameBatches(List> batche } private static SearchHit newHit(String json) { - SearchHit hit = new SearchHit(randomInt(), randomAlphaOfLength(10), Collections.emptyMap(), Collections.emptyMap()); + SearchHit hit = new SearchHit(randomInt(), randomAlphaOfLength(10)); hit.sourceRef(new BytesArray(json)); return hit; } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java index d9bcf0ee0381a..e84a11c1be598 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java @@ -921,9 +921,9 @@ private static SearchResponse createSearchResponse(List> sou fields.put("field_1", new DocumentField("field_1", Collections.singletonList("foo"))); fields.put("field_2", new DocumentField("field_2", Collections.singletonList("foo"))); - SearchHit hit = new SearchHit(123, String.valueOf(map.hashCode()), fields, Collections.emptyMap()).sourceRef( - BytesReference.bytes(XContentFactory.jsonBuilder().map(_source)) - ); + SearchHit hit = new SearchHit(123, String.valueOf(map.hashCode())); + hit.addDocumentFields(fields, Collections.emptyMap()); + hit.sourceRef(BytesReference.bytes(XContentFactory.jsonBuilder().map(_source))); list.add(hit); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java index 43fbb70d86abb..d34343a4dc200 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java @@ -189,7 +189,8 @@ public MockClientBuilder prepareSearchFields(String indexName, List fields; public SearchHitBuilder(int docId) { - fields = new HashMap<>(); - hit = new SearchHit(docId, null, fields, null); + hit = new SearchHit(docId, null); } public SearchHitBuilder addField(String name, Object value) { @@ -34,7 +30,7 @@ public SearchHitBuilder addField(String name, Object value) { } public SearchHitBuilder addField(String name, List values) { - fields.put(name, new DocumentField(name, values)); + hit.setDocumentField(name, new DocumentField(name, values)); return this; } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java index 0d6c590268d15..cc7f2fdb8128f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java @@ -300,7 +300,7 @@ private SearchHit tokenHit(int idx, BytesReference source) { final Map accessToken = (Map) sourceMap.get("access_token"); @SuppressWarnings("unchecked") final Map userToken = (Map) accessToken.get("user_token"); - final SearchHit hit = new SearchHit(idx, "token_" + userToken.get("id"), null, null); + final SearchHit hit = new SearchHit(idx, "token_" + userToken.get("id")); hit.sourceRef(source); return hit; } catch (IOException e) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java index 609ca0cdaad0a..8d025666829b9 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java @@ -1013,7 +1013,7 @@ private void mockFindTokenFromRefreshToken(String refreshToken, UserToken userTo .realmRef(realmRef) .build(false); - final SearchHit hit = new SearchHit(randomInt(), "token_" + TokenService.hashTokenString(userToken.getId()), null, null); + final SearchHit hit = new SearchHit(randomInt(), "token_" + TokenService.hashTokenString(userToken.getId())); BytesReference source = TokenService.createTokenDocument(userToken, storedRefreshToken, clientAuthentication, Instant.now()); if (refreshTokenStatus != null) { var sourceAsMap = XContentHelper.convertToMap(source, false, XContentType.JSON).v2(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStoreTests.java index 96099e2fb1582..cb4279f15316d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStoreTests.java @@ -265,9 +265,7 @@ public void testFindTokensFor() { .mapToObj( i -> new SearchHit( randomIntBetween(0, Integer.MAX_VALUE), - SERVICE_ACCOUNT_TOKEN_DOC_TYPE + "-" + accountId.asPrincipal() + "/" + tokenNames[i], - Map.of(), - Map.of() + SERVICE_ACCOUNT_TOKEN_DOC_TYPE + "-" + accountId.asPrincipal() + "/" + tokenNames[i] ) ) .toArray(SearchHit[]::new); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java index e357a45c75df9..000b4f725f7ef 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java @@ -906,7 +906,7 @@ private SearchHit[] buildHits(List sourcePrivile final SearchHit[] hits = new SearchHit[sourcePrivileges.size()]; for (int i = 0; i < hits.length; i++) { final ApplicationPrivilegeDescriptor p = sourcePrivileges.get(i); - hits[i] = new SearchHit(i, "application-privilege_" + p.getApplication() + ":" + p.getName(), null, null); + hits[i] = new SearchHit(i, "application-privilege_" + p.getApplication() + ":" + p.getName()); hits[i].sourceRef(new BytesArray(Strings.toString(p))); } return hits; diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingExtractorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingExtractorTests.java index 3a8e5e5ea7a45..bdc9130955488 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingExtractorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingExtractorTests.java @@ -29,7 +29,6 @@ import java.util.function.Supplier; import static java.util.Collections.singletonList; -import static java.util.Collections.singletonMap; import static org.elasticsearch.xpack.ql.execution.search.extractor.AbstractFieldHitExtractor.MultiValueSupport.NONE; import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE; import static org.elasticsearch.xpack.ql.util.CollectionUtils.combine; @@ -84,7 +83,8 @@ public void testGet() { double value = randomDouble(); double expected = Math.log(value); DocumentField field = new DocumentField(fieldName, singletonList(value)); - SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField(fieldName, field); assertEquals(expected, extractor.process(hit)); } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java index bfeab90bcfdee..aca64a467934d 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java @@ -32,7 +32,6 @@ import static java.util.Arrays.asList; import static java.util.Collections.singletonList; -import static java.util.Collections.singletonMap; import static org.elasticsearch.common.time.DateUtils.toMilliSeconds; import static org.elasticsearch.xpack.ql.execution.search.extractor.AbstractFieldHitExtractor.MultiValueSupport.LENIENT; import static org.elasticsearch.xpack.ql.execution.search.extractor.AbstractFieldHitExtractor.MultiValueSupport.NONE; @@ -95,7 +94,8 @@ public void testGetDottedValueWithDocValues() { } DocumentField field = new DocumentField(fieldName, documentFieldValues); - SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField(fieldName, field); Object result = documentFieldValues.isEmpty() ? null : documentFieldValues.get(0); assertEquals(result, extractor.extract(hit)); } @@ -112,7 +112,8 @@ public void testGetDocValue() { documentFieldValues.add(randomValue()); } DocumentField field = new DocumentField(fieldName, documentFieldValues); - SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField(fieldName, field); Object result = documentFieldValues.isEmpty() ? null : documentFieldValues.get(0); assertEquals(result, extractor.extract(hit)); } @@ -126,7 +127,8 @@ public void testGetDate() { ZonedDateTime zdt = DateUtils.asDateTimeWithMillis(millis, zoneId).plusNanos(nanosOnly); List documentFieldValues = Collections.singletonList(StringUtils.toString(zdt)); DocumentField field = new DocumentField("my_date_nanos_field", documentFieldValues); - SearchHit hit = new SearchHit(1, null, singletonMap("my_date_nanos_field", field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField("my_date_nanos_field", field); FieldHitExtractor extractor = new FieldHitExtractor("my_date_nanos_field", DATETIME, zoneId, LENIENT); assertEquals(zdt, extractor.extract(hit)); } @@ -142,7 +144,8 @@ public void testMultiValuedDocValue() { String fieldName = randomAlphaOfLength(5); FieldHitExtractor fe = getFieldHitExtractor(fieldName); DocumentField field = new DocumentField(fieldName, asList("a", "b")); - SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField(fieldName, field); QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extract(hit)); assertThat(ex.getMessage(), is("Arrays (returned by [" + fieldName + "]) are not supported")); } @@ -151,7 +154,8 @@ public void testExtractSourcePath() { FieldHitExtractor fe = getFieldHitExtractor("a.b.c"); Object value = randomValue(); DocumentField field = new DocumentField("a.b.c", singletonList(value)); - SearchHit hit = new SearchHit(1, null, null, singletonMap("a.b.c", field), null); + SearchHit hit = new SearchHit(1, null, null); + hit.setDocumentField("a.b.c", field); assertThat(fe.extract(hit), is(value)); } @@ -159,7 +163,8 @@ public void testMultiValuedSource() { FieldHitExtractor fe = getFieldHitExtractor("a"); Object value = randomValue(); DocumentField field = new DocumentField("a", asList(value, value)); - SearchHit hit = new SearchHit(1, null, null, singletonMap("a", field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField("a", field); QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extract(hit)); assertThat(ex.getMessage(), is("Arrays (returned by [a]) are not supported")); } @@ -169,7 +174,8 @@ public void testMultiValuedSourceAllowed() { Object valueA = randomValue(); Object valueB = randomValue(); DocumentField field = new DocumentField("a", asList(valueA, valueB)); - SearchHit hit = new SearchHit(1, null, null, singletonMap("a", field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField("a", field); assertEquals(valueA, fe.extract(hit)); } @@ -181,7 +187,8 @@ public void testGeoShapeExtraction() { map.put("coordinates", asList(1d, 2d)); map.put("type", "Point"); DocumentField field = new DocumentField(fieldName, singletonList(map)); - SearchHit hit = new SearchHit(1, null, null, singletonMap(fieldName, field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField(fieldName, field); assertEquals(new GeoShape(1, 2), fe.extract(hit)); } @@ -197,18 +204,16 @@ public void testMultipleGeoShapeExtraction() { map2.put("coordinates", asList(3d, 4d)); map2.put("type", "Point"); DocumentField field = new DocumentField(fieldName, asList(map1, map2)); - SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField(fieldName, field); QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extract(hit)); assertThat(ex.getMessage(), is("Arrays (returned by [" + fieldName + "]) are not supported")); FieldHitExtractor lenientFe = new FieldHitExtractor(fieldName, randomBoolean() ? GEO_SHAPE : SHAPE, UTC, LENIENT); - assertEquals( - new GeoShape(3, 4), - lenientFe.extract( - new SearchHit(1, null, null, singletonMap(fieldName, new DocumentField(fieldName, singletonList(map2))), null) - ) - ); + SearchHit searchHit = new SearchHit(1, "1"); + searchHit.setDocumentField(fieldName, new DocumentField(fieldName, singletonList(map2))); + assertEquals(new GeoShape(3, 4), lenientFe.extract(searchHit)); } public void testUnsignedLongExtraction() { @@ -218,7 +223,8 @@ public void testUnsignedLongExtraction() { String fieldName = randomAlphaOfLength(10); DocumentField field = new DocumentField(fieldName, singletonList(value)); - SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField(fieldName, field); FieldHitExtractor fe = new FieldHitExtractor(fieldName, UNSIGNED_LONG, randomZone(), randomBoolean() ? NONE : LENIENT); assertEquals(bi, fe.extract(hit)); @@ -231,7 +237,8 @@ public void testVersionExtraction() { String fieldName = randomAlphaOfLength(10); DocumentField field = new DocumentField(fieldName, singletonList(value)); - SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, field), null); + SearchHit hit = new SearchHit(1, null); + hit.setDocumentField(fieldName, field); FieldHitExtractor fe = new FieldHitExtractor(fieldName, VERSION, randomZone(), randomBoolean() ? NONE : LENIENT); assertEquals(version.toString(), fe.extract(hit).toString()); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/TopHitsAggExtractorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/TopHitsAggExtractorTests.java index c816544a92e2b..b36f286645efc 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/TopHitsAggExtractorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/TopHitsAggExtractorTests.java @@ -116,19 +116,14 @@ public void testExtractUnsignedLong() { private SearchHits searchHitsOf(Object value) { TotalHits totalHits = new TotalHits(10, TotalHits.Relation.EQUAL_TO); - return new SearchHits( - new SearchHit[] { - new SearchHit( - 1, - "docId", - Collections.singletonMap("topHitsAgg", new DocumentField("field", Collections.singletonList(value))), - Collections.singletonMap( - "topHitsAgg", - new DocumentField("_ignored", Collections.singletonList(randomValueOtherThan(value, () -> randomAlphaOfLength(5)))) - ) - ) }, - totalHits, - 0.0f + SearchHit searchHit = new SearchHit(1, "docId"); + searchHit.addDocumentFields( + Collections.singletonMap("topHitsAgg", new DocumentField("field", Collections.singletonList(value))), + Collections.singletonMap( + "topHitsAgg", + new DocumentField("_ignored", Collections.singletonList(randomValueOtherThan(value, () -> randomAlphaOfLength(5)))) + ) ); + return new SearchHits(new SearchHit[] { searchHit }, totalHits, 0.0f); } } diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java index 9ab58fbc17b16..ac728f59d5d3b 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java @@ -86,7 +86,7 @@ public void testExecuteWithAggs() throws Exception { public void testExecuteAccessHits() throws Exception { CompareCondition condition = new CompareCondition("ctx.payload.hits.hits.0._score", CompareCondition.Op.EQ, 1, Clock.systemUTC()); - SearchHit hit = new SearchHit(0, "1", null, null); + SearchHit hit = new SearchHit(0, "1"); hit.score(1f); hit.shard(new SearchShardTarget("a", new ShardId("a", "indexUUID", 0), null)); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java index b81e0bc5651f5..6feb977255de9 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java @@ -208,7 +208,7 @@ void stopExecutor() {} SearchHit[] hits = new SearchHit[count]; for (int i = 0; i < count; i++) { String id = String.valueOf(i); - SearchHit hit = new SearchHit(1, id, Collections.emptyMap(), Collections.emptyMap()); + SearchHit hit = new SearchHit(1, id); hit.version(1L); hit.shard(new SearchShardTarget("nodeId", new ShardId(watchIndex, 0), "whatever")); hits[i] = hit; diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java index 22f4ec6d3a74c..348f552465c3a 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java @@ -225,7 +225,7 @@ public void testFindTriggeredWatchesGoodCase() { when(searchResponse1.getSuccessfulShards()).thenReturn(1); when(searchResponse1.getTotalShards()).thenReturn(1); BytesArray source = new BytesArray("{}"); - SearchHit hit = new SearchHit(0, "first_foo", null, null); + SearchHit hit = new SearchHit(0, "first_foo"); hit.version(1L); hit.shard(new SearchShardTarget("_node_id", new ShardId(index, 0), null)); hit.sourceRef(source); @@ -240,7 +240,7 @@ public void testFindTriggeredWatchesGoodCase() { }).when(client).execute(eq(SearchAction.INSTANCE), any(), any()); // First return a scroll response with a single hit and then with no hits - hit = new SearchHit(0, "second_foo", null, null); + hit = new SearchHit(0, "second_foo"); hit.version(1L); hit.shard(new SearchShardTarget("_node_id", new ShardId(index, 0), null)); hit.sourceRef(source);