diff --git a/changelog/unreleased/SOLR-18023-mapping-field.yml b/changelog/unreleased/SOLR-18023-mapping-field.yml new file mode 100644 index 00000000000..1f06476dcc1 --- /dev/null +++ b/changelog/unreleased/SOLR-18023-mapping-field.yml @@ -0,0 +1,7 @@ +title: Field tpe that has key-value pairs as field value (i.e. mapping field) +type: added +authors: +- name: Isabelle Giguere +links: +- name: SOLR-18023 + url: https://issues.apache.org/jira/browse/SOLR-18023 \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 7b635cbbeb9..9bf6f1a8393 100644 --- a/settings.gradle +++ b/settings.gradle @@ -57,6 +57,7 @@ include "solr:modules:ltr" include "solr:modules:s3-repository" include "solr:modules:scripting" include "solr:modules:sql" +include "solr:modules:mapping-field" include "solr:webapp" include "solr:benchmark" include "solr:test-framework" diff --git a/solr/modules/mapping-field/README.md b/solr/modules/mapping-field/README.md new file mode 100644 index 00000000000..4c8651da6f3 --- /dev/null +++ b/solr/modules/mapping-field/README.md @@ -0,0 +1,30 @@ + + +Welcome to Apache Solr's Mapping Field module! +======== + +> [!CAUTION] +> This feature is currently experimental. + +Apache Solr Mapping Field allows indexing metadata in the form of key-value pairs under a single heading. +The mappings QParser plugin (MappingsQParserPlugin) allows searching for specific keys and values within mappings. + +# Getting Started With Solr Mapping Field + +For information on how to get started with Solr Mapping Field please see: + * [Solr Reference Guide's section on Mapping Field](https://solr.apache.org/guide/solr/latest/query-guide/mapping-field.html) diff --git a/solr/modules/mapping-field/build.gradle b/solr/modules/mapping-field/build.gradle new file mode 100644 index 00000000000..2f71cd9f20c --- /dev/null +++ b/solr/modules/mapping-field/build.gradle @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply plugin: 'java-library' + +description = 'mapping-field plugin' + +dependencies { + implementation project(':solr:core') + implementation project(':solr:solrj') + implementation libs.apache.lucene.core + implementation libs.slf4j.api + + testImplementation project(':solr:test-framework') + testImplementation libs.carrotsearch.randomizedtesting.runner + testImplementation libs.junit.junit +} + diff --git a/solr/modules/mapping-field/src/java/org/apache/solr/mappings/search/MappingsQParserPlugin.java b/solr/modules/mapping-field/src/java/org/apache/solr/mappings/search/MappingsQParserPlugin.java new file mode 100644 index 00000000000..f2b70994007 --- /dev/null +++ b/solr/modules/mapping-field/src/java/org/apache/solr/mappings/search/MappingsQParserPlugin.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.mappings.search; + +import java.lang.invoke.MethodHandles; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.WildcardQuery; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.MappingType; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.QParser; +import org.apache.solr.search.QParserPlugin; +import org.apache.solr.search.QueryParsing; +import org.apache.solr.search.SyntaxError; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Plugin to allow searching in mapping fields keys and values. + * + *

Mappings Query Parser Syntax: q={!mappings f=my_mapping_field key="some key" value=123} + */ +public class MappingsQParserPlugin extends QParserPlugin { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + /** Name of the 'mappings' QParserPlugin */ + public static final String NAME = "mappings"; + + /** mappings QPArser: Local param 'field' */ + public static final String FIELD_PARAM = QueryParsing.F; + + /** mappings QPArser: Local param 'key' */ + public static final String KEY_PARAM = "key"; + + /** mappings QPArser: Local param 'value' */ + public static final String VALUE_PARAM = "value"; + + @Override + public QParser createParser( + String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + return new MappingsQParser(qstr, localParams, params, req); + } + + /** Parser for 'mappings' query */ + public class MappingsQParser extends QParser { + + public MappingsQParser( + String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + super(qstr, localParams, params, req); + } + + @Override + public Query parse() throws SyntaxError { + if (log.isDebugEnabled()) { + log.debug("Parse local params: {}", localParams.toQueryString()); + } + String searchField = localParams.get(FIELD_PARAM); + String keySearch = localParams.get(KEY_PARAM); + String valueSearch = localParams.get(VALUE_PARAM); + + if (keySearch == null && valueSearch == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Mappings query must specify 'key' and/or 'value'."); + } + + if (searchField == null || searchField.equals("*")) { + return searchAnyMapping(keySearch, valueSearch); + + } else if (!(req.getSchema().getFieldType(searchField) instanceof MappingType)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Mappings query 'f' parameter must specify the name of a MappingType field."); + } + + return searchRequiredMapping(searchField, keySearch, valueSearch); + } + + /** Search or the key and/or value where the mapping exists */ + private Query searchRequiredMapping(String searchField, String keySearch, String valueSearch) { + SchemaField sf = req.getSchema().getField(searchField); + FieldType ft = req.getSchema().getFieldType(searchField); + + BooleanQuery.Builder boolQuery = new BooleanQuery.Builder(); + + MappingType mappingType = (MappingType) ft; + Query mappingQuery = ft.getExistenceQuery(this, sf); + boolQuery.add(mappingQuery, BooleanClause.Occur.MUST); + + addSubQuery(boolQuery, BooleanClause.Occur.MUST, mappingType.getKeyField(sf), keySearch); + addSubQuery(boolQuery, BooleanClause.Occur.MUST, mappingType.getValueField(sf), valueSearch); + + BooleanQuery query = boolQuery.build(); + if (log.isDebugEnabled()) { + log.debug("Parsed query: {}", query.toString()); + } + return query; + } + + /** Search for key and/or value in any mapping */ + private Query searchAnyMapping(String keySearch, String valueSearch) { + // all mappings to create key and value fields + BooleanQuery.Builder boolQuery = new BooleanQuery.Builder(); + req.getSchema().getFields().entrySet().stream() + .filter(e -> (e.getValue().getType() instanceof MappingType)) + .forEach( + e -> { + String mappingField = e.getKey(); + SchemaField sf = req.getSchema().getField(mappingField); + FieldType ft = req.getSchema().getFieldType(mappingField); + MappingType mappingType = (MappingType) ft; + + addSubQuery( + boolQuery, BooleanClause.Occur.SHOULD, mappingType.getKeyField(sf), keySearch); + addSubQuery( + boolQuery, + BooleanClause.Occur.SHOULD, + mappingType.getValueField(sf), + valueSearch); + }); + + BooleanQuery query = boolQuery.build(); + if (log.isDebugEnabled()) { + log.debug("Parsed query: {}", query.toString()); + } + return query; + } + + /** create and add a sub-query, for key or value */ + private void addSubQuery( + BooleanQuery.Builder boolQuery, + BooleanClause.Occur occur, + SchemaField field, + String search) { + if (log.isDebugEnabled()) { + log.debug("Create sub-query for: {}", search); + } + if (search != null) { + if (search.equals("*")) { + Query existQ = field.getType().getExistenceQuery(this, field); + boolQuery.add(existQ, occur); + return; + } else { + if (search.startsWith("[") && search.endsWith("]")) { + String decoded = + URLDecoder.decode(search.substring(1, search.length() - 1), StandardCharsets.UTF_8); + String[] parts = decoded.split("TO"); + String min = parts[0].trim(); + String max = parts[1].trim(); + Query rangeQ = + field + .getType() + .getRangeQuery( + this, + field, + "*".equals(min) ? null : min, + "*".equals(max) ? null : max, + true, + true); + boolQuery.add(rangeQ, occur); + return; + } else if (search.contains("*") || search.contains("?")) { + Query termQ = new WildcardQuery(new Term(field.getName(), search)); + boolQuery.add(termQ, occur); + return; + } else { + Query termQ = field.getType().getFieldTermQuery(this, field, search); + boolQuery.add(termQ, occur); + } + } + } + } + } +} diff --git a/solr/modules/mapping-field/src/java/org/apache/solr/schema/MappingType.java b/solr/modules/mapping-field/src/java/org/apache/solr/schema/MappingType.java new file mode 100644 index 00000000000..27a54986c79 --- /dev/null +++ b/solr/modules/mapping-field/src/java/org/apache/solr/schema/MappingType.java @@ -0,0 +1,383 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.schema; + +import java.io.IOException; +import java.io.StringReader; +import java.io.Writer; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.params.MapSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.JsonTextWriter; +import org.apache.solr.internal.csv.CSVParser; +import org.apache.solr.response.JSONWriter; +import org.apache.solr.response.TextResponseWriter; +import org.apache.solr.response.XMLWriter; +import org.apache.solr.search.QParser; +import org.apache.solr.uninverting.UninvertingReader.Type; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class allows indexing key-value pairs. Only subclasses of {@link PrimitiveFieldType} are + * supported for keys and values. + * + *

Define the mapping value type using {@code subFieldSuffix} or {@code subFieldType}, as for any + * subclass of {@link AbstractSubTypeFieldType}. The key type is the first found {@link StrField} by + * default, or it can be defined using {@code keyFieldSuffix} or {@code keyFieldType}. + * + *

Expected input document would have a CSV field value:
+ * {@code "key","value"} + * + *

XML output document will show the mapping key as the {@code name} attribute of the type + * element of the value. A wrapping {@code } XML element helps differentiate mappings from + * regular Solr elements (i.e.: {@code }, {@code }, etc). Refer to unit tests in {@code + * TestMappingType} for examples. + * + *

Json outputs only strings, for any type: refer to {@link JSONWriter}, {@link JsonTextWriter}. + */ +public class MappingType extends AbstractSubTypeFieldType { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + protected static final int KEY = 0; + protected static final int VALUE = 1; + + private static final int KEY_VALUE_SIZE = 2; + + private static final String KEY_ATTR = "key"; + private static final String VAL_ATTR = "value"; + + private static final String KEY_FIELD_SUFFIX = "keyFieldSuffix"; + private static final String KEY_FIELD_TYPE = "keyFieldType"; + + private String keyFieldType = null; + private String keySuffix = null; + private FieldType keyType = null; + + @Override + protected void init(IndexSchema schema, Map args) { + super.init(schema, args); + + if (!(subType instanceof PrimitiveFieldType)) { + throw new UnsupportedOperationException( + "Unsupported value type in MappingType: " + subType.getClass().getName()); + } + + SolrParams p = new MapSolrParams(args); + keyFieldType = p.get(KEY_FIELD_TYPE); + keySuffix = p.get(KEY_FIELD_SUFFIX); + if (keyFieldType != null) { + args.remove(KEY_FIELD_TYPE); + keyType = schema.getFieldTypeByName(keyFieldType.trim()); + keySuffix = POLY_FIELD_SEPARATOR + keyType.typeName; + } else if (keySuffix != null) { + args.remove(KEY_FIELD_SUFFIX); + keyType = schema.getDynamicFieldType("keyField_" + keySuffix); + } else { + String strFieldType = + schema.getFieldTypes().entrySet().stream() + .filter(e -> (e.getValue() instanceof StrField)) + .findFirst() + .get() + .getKey(); + keyType = schema.getFieldTypeByName(strFieldType); + keySuffix = POLY_FIELD_SEPARATOR + keyType.typeName; + } + + createSuffixCache(KEY_VALUE_SIZE); + } + + @Override + protected void createSuffixCache(int size) { + suffixes = new String[size]; + suffixes[KEY] = "_key" + keySuffix; + suffixes[VALUE] = "_value" + suffix; + } + + @Override + public void inform(IndexSchema schema) { + this.schema = schema; + if (keyType != null) { + SchemaField protoKey = registerDynamicPrototype(schema, keyType, keySuffix, this); + dynFieldProps += protoKey.getProperties(); + } + if (subType != null) { + SchemaField protoVal = registerDynamicPrototype(schema, subType, suffix, this); + dynFieldProps += protoVal.getProperties(); + } + } + + private SchemaField registerDynamicPrototype( + IndexSchema schema, FieldType type, String fieldSuffix, FieldType polyField) { + String name = "*" + fieldSuffix; + Map props = new HashMap<>(); + props.put("indexed", "true"); + log.warn( + "MappingType requires indexed subtypes (key, value). " + + "Setting the subtype '{}' to 'indexed=true'", + type.getTypeName()); + + props.put("stored", "false"); + log.warn( + "MappingType does not support stored on the subtypes (key, value). " + + "Setting the subtype '{}' to 'stored=false'", + type.getTypeName()); + + props.put("multiValued", "false"); + log.warn( + "MappingType does not support multiValued on the subtypes (key, value). " + + "Setting the subtype '{}' to 'multiValued=false'", + type.getTypeName()); + + props.put("docValues", "false"); + log.warn( + "MappingType does not support docValues on the subtypes (key, value). " + + "Setting the subtype '{}' to 'docValues=false'", + type.getTypeName()); + + int p = SchemaField.calcProps(name, type, props); + SchemaField proto = SchemaField.create(name, type, p, null); + schema.registerDynamicFields(proto); + return proto; + } + + @Override + protected void checkSupportsDocValues() { + // DocValues supported only when enabled at the fieldType + if (!hasProperty(DOC_VALUES)) { + throw new UnsupportedOperationException( + "MappingType can't have docValues=true in the field definition, use docValues=true in the fieldType definition."); + } + } + + @Override + public boolean isPolyField() { + return true; // really only true if the field is indexed + } + + @Override + public List createFields(SchemaField field, Object value) { + String externalVal = value.toString(); + String[] csv = parseCommaSeparatedList(externalVal); + + List fields = new ArrayList<>((suffixes.length * 2) + 1); + + if (field.indexed()) { + SchemaField keyField = getKeyField(field); + fields.addAll(keyField.createFields(csv[KEY])); + + SchemaField valField = getValueField(field); + fields.addAll(valField.createFields(csv[VALUE])); + } + + if (field.stored()) { + fields.add(createField(field.getName(), externalVal, StoredField.TYPE)); + } + + if (field.hasDocValues()) { + fields.add(createDocValuesField(field, value.toString())); + } + + return fields; + } + + private IndexableField createDocValuesField(SchemaField field, String value) { + IndexableField docval; + final BytesRef bytes = new BytesRef(toInternal(value)); + if (field.multiValued()) { + docval = new SortedSetDocValuesField(field.getName(), bytes); + } else { + docval = new SortedDocValuesField(field.getName(), bytes); + } + return docval; + } + + @Override + public IndexableField createField(SchemaField field, Object value) { + throw new UnsupportedOperationException( + "MappingType uses multiple fields. field=" + field.getName()); + } + + /** + * Given a string of comma-separated values, return a String array of length 2 containing the + * values. + * + * @param externalVal The value to parse + * @return An array of the values that make up the mapping + * @throws SolrException if the input value cannot be parsed + */ + protected static String[] parseCommaSeparatedList(String externalVal) throws SolrException { + String[] out = new String[KEY_VALUE_SIZE]; + // input is: "key","value" + CSVParser parser = new CSVParser(new StringReader(externalVal)); + try { + String[] tokens = parser.getLine(); + if (tokens != null && tokens.length > 0) { + out[0] = tokens[0]; + out[1] = tokens.length > 1 ? tokens[1] : ""; + + } else { + throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid input value: " + externalVal); + } + } catch (IOException e) { + throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to parse as CSV: " + externalVal); + } + return out; + } + + @Override + public void write(TextResponseWriter writer, String name, IndexableField f) throws IOException { + if (log.isTraceEnabled()) { + log.trace( + "Write MappingType '{}' as 'mapping' from indexable '{}'", + name, + f.getClass().getSimpleName()); + } + String[] keyVal = parseCommaSeparatedList(f.stringValue()); + if (writer instanceof XMLWriter) { + doWriteXMl(writer, name, keyVal[KEY], keyVal[VALUE]); + } + if (writer instanceof JSONWriter) { + doWriteJson(writer, name, keyVal[KEY], keyVal[VALUE]); + } + } + + private void doWriteJson(TextResponseWriter writer, String name, String key, String val) + throws IOException { + Map map = new HashMap<>(); + map.put(KEY_ATTR, key); + map.put(VAL_ATTR, val); + // JSONWriter, JsonTextWriter : write everything as str. + writer.writeMap(name, map, false, true); + } + + private void doWriteXMl(TextResponseWriter writer, String name, String key, String val) + throws IOException { + Writer w = writer.getWriter(); + if (writer.doIndent()) { + writer.indent(); + writer.incLevel(); + } + w.write(""); + writeSubFieldObject(writer, KEY_ATTR, key, keyType.getClass().getSimpleName()); + writeSubFieldObject(writer, VAL_ATTR, val, subType.getClass().getSimpleName()); + + if (writer.doIndent()) { + writer.decLevel(); + writer.indent(); + } + w.write(""); + } + + private void writeSubFieldObject( + TextResponseWriter writer, String key, String val, String subTypeClsName) throws IOException { + switch (subTypeClsName) { + case "StrField": + writer.writeStr(key, val, true); + break; + case "IntPointField": + case "TrieIntField": + writer.writeInt(key, val); + break; + case "LongPointField": + case "TrieLongField": + writer.writeLong(key, val); + break; + case "FloatPointField": + case "TrieFloatField": + writer.writeFloat(key, val); + break; + case "DoublePointField": + case "TrieDoubleField": + writer.writeDouble(key, val); + break; + case "DatePointField": + case "TrieDateField": + writer.writeDate(key, val); + break; + case "BoolField": + writer.writeBool(key, val); + break; + default: + throw new UnsupportedOperationException( + "Unsupported value type in MappingType: " + subTypeClsName); + } + } + + /** Sorting is not supported on {@code MappingType} */ + @Override + public SortField getSortField(SchemaField field, boolean top) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Sorting not supported on MappingType " + field.getName()); + } + + /** Uninversion is not supported on {@code MappingType} */ + @Override + public Type getUninversionType(SchemaField sf) { + return null; + } + + /** Range query is not supported on {@code MappingType} */ + @Override + protected Query getSpecializedRangeQuery( + QParser parser, + SchemaField field, + String part1, + String part2, + boolean minInclusive, + boolean maxInclusive) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Range query not supported on MappingType " + field.getName()); + } + + public SchemaField getKeyField(SchemaField base) { + return schema.getField(base.getName() + suffixes[KEY]); + } + + public SchemaField getValueField(SchemaField base) { + return schema.getField(base.getName() + suffixes[VALUE]); + } + + /** Returns a Query to support existence search on the mapping field name */ + @Override + public Query getFieldQuery(QParser parser, SchemaField field, String externalVal) { + BytesRefBuilder br = new BytesRefBuilder(); + readableToIndexed(externalVal, br); + return new TermQuery(new Term(field.getName(), br)); + } +} diff --git a/solr/modules/mapping-field/src/java/overview.html b/solr/modules/mapping-field/src/java/overview.html new file mode 100644 index 00000000000..77a2fb8adb2 --- /dev/null +++ b/solr/modules/mapping-field/src/java/overview.html @@ -0,0 +1,62 @@ + + + +Apache Solr Search Server: Mapping Field module + +

+This module contains logic to index and search key-value mappings in a single field in Solr. +

+

+Solr allows indexing single keys (i.e.: field names) to one or many values (i.e.: field value). However, in some use cases, +it can be useful for the field value itself to be a key-value mapping, and for such mapping to be indexed as one field. +For example, if a document goes through a classification engine before indexing, the results may include the name of the +taxonomy, the document's classification(s), and some sort of relevance or confidence score per classification. +In such a use case, when multiple taxonomies are used, the indexed metadata can become unusable if results from all taxonomies +are indexed as separate fields (as with Solr's other field types). +

+

+This module allows indexing data as key-value mappings with a single heading: +{@code + + + classification1 + 98.5 + + + classification1 + 82.6 + + +} +

+

+When searching, a local params query can be used to match specific keys and/or values in a given mapping, or across any +mapping fields. Note that a range query must be wrapped in quotes. +

+

+

Code structure

+

+The code consists of a field type {@link org.apache.solr.schema.MappingType}, which is an extension of +{@link org.apache.solr.schema.AbstractSubTypeFieldType}, and it's corresponding parser plugin and parser +implementations: {@link org.apache.solr.mappings.search.MappingsQParserPlugin}, and +{@link org.apache.solr.mappings.search.MappingsQParserPlugin.MappingsQParser}. + + diff --git a/solr/modules/mapping-field/src/test-files/log4j2.xml b/solr/modules/mapping-field/src/test-files/log4j2.xml new file mode 100644 index 00000000000..528299e3e0b --- /dev/null +++ b/solr/modules/mapping-field/src/test-files/log4j2.xml @@ -0,0 +1,40 @@ + + + + + + + + + %maxLen{%-4r %-5p (%t) [%notEmpty{n:%X{node_name}}%notEmpty{ c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}%notEmpty{ t:%X{trace_id}}] %c{1.} %m%notEmpty{ + =>%ex{short}}}{10240}%n + + + + + + + + + + + + + + + diff --git a/solr/modules/mapping-field/src/test-files/solr/collection1/conf/schema-mappings.xml b/solr/modules/mapping-field/src/test-files/solr/collection1/conf/schema-mappings.xml new file mode 100644 index 00000000000..f2fc5ff507c --- /dev/null +++ b/solr/modules/mapping-field/src/test-files/solr/collection1/conf/schema-mappings.xml @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + diff --git a/solr/modules/mapping-field/src/test-files/solr/collection1/conf/solrconfig-mappings.xml b/solr/modules/mapping-field/src/test-files/solr/collection1/conf/solrconfig-mappings.xml new file mode 100644 index 00000000000..5af6393ed03 --- /dev/null +++ b/solr/modules/mapping-field/src/test-files/solr/collection1/conf/solrconfig-mappings.xml @@ -0,0 +1,30 @@ + + + + + + + ${tests.luceneMatchVersion:LATEST} + ${solr.data.dir:} + + + + + + diff --git a/solr/modules/mapping-field/src/test-files/solr/solr.xml b/solr/modules/mapping-field/src/test-files/solr/solr.xml new file mode 100644 index 00000000000..7506c1c8951 --- /dev/null +++ b/solr/modules/mapping-field/src/test-files/solr/solr.xml @@ -0,0 +1,41 @@ + + + + + + ${shareSchema:false} + ${configSetBaseDir:configsets} + ${coreRootDirectory:.} + + + ${urlScheme:} + ${socketTimeout:90000} + ${connTimeout:15000} + + + + 127.0.0.1 + ${hostPort:8983} + ${solr.zkclienttimeout:30000} + ${genericCoreNodeNames:true} + ${leaderVoteWait:10000} + ${distribUpdateConnTimeout:45000} + ${distribUpdateSoTimeout:340000} + + + diff --git a/solr/modules/mapping-field/src/test/org/apache/solr/mappings/MappingsTestUtils.java b/solr/modules/mapping-field/src/test/org/apache/solr/mappings/MappingsTestUtils.java new file mode 100644 index 00000000000..6b858c76b4c --- /dev/null +++ b/solr/modules/mapping-field/src/test/org/apache/solr/mappings/MappingsTestUtils.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.mappings; + +import com.carrotsearch.randomizedtesting.generators.RandomStrings; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.Random; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.schema.NumberType; + +/** Utilities for mapping-field module tests */ +public class MappingsTestUtils { + + private static final SimpleDateFormat format = + new SimpleDateFormat("yyy-MM-dd'T'HH:mm:ss'Z'", Locale.ENGLISH); + + /** generate string-string mappings */ + public static List generateDocs( + Random random, String field, int nb, boolean multiVal, boolean predictableStrKey) { + return generateDocs(random, field, nb, multiVal, null, null, predictableStrKey); + } + + /** generate string-NumberType mappings */ + public static List generateDocs( + Random random, + String field, + int nb, + boolean multiVal, + NumberType subType, + boolean predictableStrKey) { + return generateDocs(random, field, nb, multiVal, subType, null, predictableStrKey); + } + + /** generate solr input documents with the given keyType and subType (value type) */ + public static List generateDocs( + Random random, + String field, + int nb, + boolean multiVal, + NumberType subType, + NumberType keyType, + boolean predictableStrKey) { + List docs = new ArrayList<>(); + if (multiVal) { + for (int i = 0; i < nb; i++) { + SolrInputDocument sdoc = new SolrInputDocument(); + sdoc.addField("id", "" + i); + for (int j = 0; j < nb; j++) { + String key = null; + if (predictableStrKey) { + key = "key_" + i + "_" + j; + } else { + key = getRandomValue(keyType, random); + } + String val = getRandomValue(subType, random); + sdoc.addField(field, "\"" + key + "\",\"" + val + "\""); + } + docs.add(sdoc); + } + } else { + for (int i = 0; i < nb; i++) { + SolrInputDocument sdoc = new SolrInputDocument(); + sdoc.addField("id", "" + i); + String key = null; + if (predictableStrKey) { + key = "key_" + i; + } else { + key = getRandomValue(keyType, random); + } + String val = getRandomValue(subType, random); + sdoc.addField(field, "\"" + key + "\",\"" + val + "\""); + docs.add(sdoc); + } + } + return docs; + } + + private static String getRandomValue(NumberType nbType, Random random) { + String str = null; + + if (nbType != null) { + Double dbl = random.nextDouble() * 10; + switch (nbType) { + case NumberType.INTEGER: + str = String.valueOf(dbl.intValue()); + break; + case NumberType.LONG: + str = String.valueOf(dbl.longValue()); + break; + case NumberType.FLOAT: + str = String.valueOf(dbl.floatValue()); + break; + case NumberType.DATE: + Instant instant = + Instant.ofEpochSecond(random.nextInt(0, (int) Instant.now().getEpochSecond())); + Date dt = Date.from(instant); + str = format.format(dt); + break; + default: + str = String.valueOf(dbl.doubleValue()); + break; + } + } else { + str = RandomStrings.randomAsciiAlphanumOfLengthBetween(random, 5, 10); + } + return str; + } +} diff --git a/solr/modules/mapping-field/src/test/org/apache/solr/mappings/search/TestMappingsQParserPlugin.java b/solr/modules/mapping-field/src/test/org/apache/solr/mappings/search/TestMappingsQParserPlugin.java new file mode 100644 index 00000000000..d54b80382c7 --- /dev/null +++ b/solr/modules/mapping-field/src/test/org/apache/solr/mappings/search/TestMappingsQParserPlugin.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.mappings.search; + +import com.carrotsearch.randomizedtesting.generators.RandomStrings; +import java.lang.invoke.MethodHandles; +import java.util.List; +import org.apache.lucene.search.Query; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.mappings.MappingsTestUtils; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrQueryRequestBase; +import org.apache.solr.search.QParser; +import org.apache.solr.search.QueryParsing; +import org.apache.solr.search.SyntaxError; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestMappingsQParserPlugin extends SolrTestCaseJ4 { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig-mappings.xml", "schema-mappings.xml"); + } + + @Test + public void testQParserKeySearch() { + String queryStr = """ + q={!mappings f=multi_mapping key="key_1" value=*} + """; + ModifiableSolrParams localParams = new ModifiableSolrParams(); + localParams.add(MappingsQParserPlugin.FIELD_PARAM, "multi_mapping"); + localParams.add(QueryParsing.TYPE, "mappings"); + localParams.add(MappingsQParserPlugin.KEY_PARAM, "key_1"); + localParams.add(MappingsQParserPlugin.VALUE_PARAM, "*"); + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.add(CommonParams.Q, queryStr); + + MappingsQParserPlugin parserPlugin = new MappingsQParserPlugin(); + + SolrQueryRequest req = new SolrQueryRequestBase(h.getCore(), params) {}; + + QParser parser = parserPlugin.createParser(queryStr, localParams, params, req); + + try { + Query query = parser.parse(); + if (log.isInfoEnabled()) { + log.info(query.toString()); + } + Assert.assertEquals( + "+FieldExistsQuery [field=multi_mapping] +multi_mapping_key___string:key_1 +multi_mapping_value___string:{* TO *}", + query.toString()); + } catch (SyntaxError e) { + Assert.fail("Should not throw SyntaxError"); + } + } + + @Test + public void testQParserValueSearch() { + String queryStr = """ + q={!mappings f=float_mapping value=12.34} + """; + ModifiableSolrParams localParams = new ModifiableSolrParams(); + localParams.add(MappingsQParserPlugin.FIELD_PARAM, "float_mapping"); + localParams.add(QueryParsing.TYPE, "mappings"); + localParams.add(MappingsQParserPlugin.VALUE_PARAM, "12.34"); + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.add(CommonParams.Q, queryStr); + + MappingsQParserPlugin parserPlugin = new MappingsQParserPlugin(); + + SolrQueryRequest req = new SolrQueryRequestBase(h.getCore(), params) {}; + + QParser parser = parserPlugin.createParser(queryStr, localParams, params, req); + + try { + Query query = parser.parse(); + if (log.isInfoEnabled()) { + log.info(query.toString()); + } + Assert.assertEquals( + "+FieldExistsQuery [field=float_mapping] +float_mapping_value___float:[12.34 TO 12.34]", + query.toString()); + } catch (SyntaxError e) { + Assert.fail("Should not throw SyntaxError"); + } + } + + @Test + public void testQParserKeySearchAnyMapping() { + String queryStr = """ + q={!mappings key="key_1"} + """; + ModifiableSolrParams localParams = new ModifiableSolrParams(); + localParams.add(QueryParsing.TYPE, "mappings"); + // use a date as key, to have a parseable key for "date_str_mapping" + localParams.add(MappingsQParserPlugin.KEY_PARAM, "2025-12-08T00:00:00Z"); + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.add(CommonParams.Q, queryStr); + + MappingsQParserPlugin parserPlugin = new MappingsQParserPlugin(); + + SolrQueryRequest req = new SolrQueryRequestBase(h.getCore(), params) {}; + + QParser parser = parserPlugin.createParser(queryStr, localParams, params, req); + + try { + Query query = parser.parse(); + if (log.isInfoEnabled()) { + log.info(query.toString()); + } + Assert.assertEquals( + "multi_mapping_key___string:2025-12-08T00:00:00Z single_mapping_key___string:2025-12-08T00:00:00Z float_mapping_key___string:2025-12-08T00:00:00Z date_str_mapping_key___date:[1765152000000 TO 1765152000000]", + query.toString()); + } catch (SyntaxError e) { + Assert.fail("Should not throw SyntaxError"); + } + } + + @Test + public void testSearchWithParsedQuery() { + String queryStr = """ + {!mappings f=single_mapping key="key_1"} + """; + int requiredDocs = 5; + List docs = + MappingsTestUtils.generateDocs(random(), "single_mapping", requiredDocs, false, true); + for (SolrInputDocument doc : docs) { + assertU(adoc(doc)); + } + assertU(commit()); + + String response = + assertXmlQ( + req("q", queryStr.trim(), "indent", "true"), + "//doc/mapping[@name=\"single_mapping\"]/str[@name=\"key\"][text()='key_1']", + "//result[@name=\"response\"][@numFound=\"1\"]"); + if (log.isInfoEnabled()) { + log.info("Parsed query response: {}", response); + } + } + + @Test + public void testSearchWithValueRangeQuery() throws Exception { + int requiredDocs = 5; + for (int i = 0; i <= requiredDocs; i++) { + SolrInputDocument sdoc = new SolrInputDocument(); + sdoc.addField("id", "" + i); + String key = RandomStrings.randomAsciiAlphanumOfLengthBetween(random(), 5, 10); + float val = (float) (i * 10); + sdoc.addField("float_mapping", "\"" + key + "\",\"" + val + "\""); + assertU(adoc(sdoc)); + } + assertU(commit()); + + // URL-encoded range query + String queryStr1 = """ + {!mappings f=float_mapping value=[30.0+TO+*]} + """; + String response1 = + assertJQ( + req("q", queryStr1.trim(), "indent", "true", "wt", "json"), + "/response/numFound==3", + "/response/docs/[0]/float_mapping/value==\"30.0\"", + "/response/docs/[1]/float_mapping/value==\"40.0\"", + "/response/docs/[2]/float_mapping/value==\"50.0\""); + if (log.isInfoEnabled()) { + log.info("Value range query response: {}", response1); + } + + // wrap the range query in quotes + String queryStr2 = """ + {!mappings f=float_mapping value="[30.0 TO *]"} + """; + assertJQ( + req("q", queryStr2.trim(), "indent", "true", "wt", "json"), + "/response/numFound==3", + "/response/docs/[0]/float_mapping/value==\"30.0\"", + "/response/docs/[1]/float_mapping/value==\"40.0\"", + "/response/docs/[2]/float_mapping/value==\"50.0\""); + } + + @Test + public void testSearchWithWildcardQuery() throws Exception { + // wildcard query without quotes + String queryStr1 = """ + {!mappings f=single_mapping key=key*} + """; + int requiredDocs = 5; + List docs1 = + MappingsTestUtils.generateDocs(random(), "single_mapping", requiredDocs, false, true); + for (SolrInputDocument doc : docs1) { + assertU(adoc(doc)); + } + assertU(commit()); + + String response1 = + assertXmlQ( + req("q", queryStr1.trim(), "indent", "true"), + "//result[@name=\"response\"][@numFound=\"5\"]"); + if (log.isInfoEnabled()) { + log.info("Wildcard query response 1: {}", response1); + } + + // wildcard query in quotes + String queryStr2 = """ + {!mappings f=single_mapping key="key?1"} + """; + List docs2 = + MappingsTestUtils.generateDocs(random(), "single_mapping", requiredDocs, false, true); + for (SolrInputDocument doc : docs2) { + assertU(adoc(doc)); + } + assertU(commit()); + + String response2 = + assertXmlQ( + req("q", queryStr2.trim(), "indent", "true"), + "//result[@name=\"response\"][@numFound=\"1\"]"); + if (log.isInfoEnabled()) { + log.info("Wildcard query response 2: {}", response2); + } + } +} diff --git a/solr/modules/mapping-field/src/test/org/apache/solr/schema/TestMappingType.java b/solr/modules/mapping-field/src/test/org/apache/solr/schema/TestMappingType.java new file mode 100644 index 00000000000..792654fd858 --- /dev/null +++ b/solr/modules/mapping-field/src/test/org/apache/solr/schema/TestMappingType.java @@ -0,0 +1,325 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.schema; + +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.SolrInputField; +import org.apache.solr.mappings.MappingsTestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Test {@link MappingType} */ +public class TestMappingType extends SolrTestCaseJ4 { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig-mappings.xml", "schema-mappings.xml"); + } + + @After + public void tearDown() throws Exception { + clearIndex(); + assertU(commit()); + super.tearDown(); + } + + @Test + public void testMappingsSchema() { + IndexSchema schema = h.getCore().getLatestSchema(); + + Map fieldTypes = schema.getFieldTypes(); + Assert.assertEquals("Wrong number of fieldType", 11, fieldTypes.size()); + Map mappingTypes = + fieldTypes.entrySet().stream() + .filter(e -> e.getKey().equals("mapping")) + .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())); + Assert.assertTrue( + "Mappings should all be MappingType", + mappingTypes.entrySet().stream().allMatch(e -> (e.getValue() instanceof MappingType))); + + Map mappingFields = + schema.getFields().entrySet().stream() + .filter(e -> e.getKey().endsWith("mapping")) + .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())); + Assert.assertEquals("Wrong number of mapping fields", 4, mappingFields.size()); + Assert.assertTrue( + "Mapping fields should all be MappingType", + mappingFields.entrySet().stream() + .allMatch(e -> (e.getValue().getType() instanceof MappingType))); + Assert.assertTrue( + "Mapping types should all be PolyFields", + mappingFields.entrySet().stream().allMatch(e -> e.getValue().getType().isPolyField())); + Assert.assertTrue( + "Mapping fields should all be PolyFields", + mappingFields.entrySet().stream().allMatch(e -> e.getValue().isPolyField())); + } + + @Test + public void testSingleValuedXml() { + /* + * key_2vLvdE MmVaK + */ + int requiredDocs = 5; + Map mappings = doAddDocs("single_mapping", requiredDocs, false, false); + + String findKeyFormat = + "//doc/mapping[@name=\"single_mapping\"]/str[@name=\"key\"][text()='%s']"; + String findValueFormat = "/parent::mapping/str[@name=\"value\"][text()='%s']"; + + String[] tests = + mappings.entrySet().stream() + .map( + e -> + String.format(Locale.ENGLISH, findKeyFormat, e.getKey()) + + String.format(Locale.ENGLISH, findValueFormat, e.getValue())) + .toList() + .toArray(new String[0]); + + String response = assertXmlQ(req("q", "*:*", "indent", "true"), tests); + log.info(response); + } + + @Test + public void testSingleValuedJson() throws Exception { + /* + * "single_mapping":{ "key":"key_0", "value":"Bqxsd" } + */ + int requiredDocs = 5; + Map mappings = doAddDocs("single_mapping", requiredDocs, false, true); + + String findKeyFormat = "/response/docs/[%d]/single_mapping/key==\"key_%d\""; + String findValueFormat = "/response/docs/[%d]/single_mapping/value==\"%s\""; + + String[] tests = new String[mappings.size() * 2]; + + for (int i = 0, j = 0; i < mappings.size(); i++, j++) { + tests[i] = String.format(Locale.ENGLISH, findKeyFormat, j, j); + tests[i++] = String.format(Locale.ENGLISH, findValueFormat, j, mappings.get("key_" + j)); + } + + String response = assertJQ(req("q", "*:*", "indent", "true", "wt", "json"), tests); + log.info(response); + } + + @Test + public void testMultiValuedXml() { + /* + * key_0vLvdE MmVaK_value_0 + * + */ + int requiredDocs = 5; + Map mappings = doAddDocs("multi_mapping", requiredDocs, true, false); + + String findKeyFormat = + "//doc/arr[@name=\"multi_mapping\"]/mapping[@name=\"%s\"]/str[@name=\"key\"][text()='%s']"; + String findValueFormat = "/parent::mapping/str[@name=\"value\"][text()='%s']"; + + String[] tests = + mappings.entrySet().stream() + .map( + e -> + String.format(Locale.ENGLISH, findKeyFormat, e.getKey(), e.getKey()) + + String.format(Locale.ENGLISH, findValueFormat, e.getValue())) + .toList() + .toArray(new String[0]); + + String response = assertXmlQ(req("q", "*:*", "indent", "true"), tests); + log.info(response); + } + + @Test + public void testMultiValuedJson() throws Exception { + /* + * "multi_mapping":[{ "key":"key_0", "value":"mPfsP_value_0" },{ "key":"key_1", + * "value":"mPfsP_value_1" }] + */ + int required = 5; + Map mappings = doAddDocs("multi_mapping", required, true, true); + + String findKeyFormat = "/response/docs/[%d]/multi_mapping/[%d]/key==\"key_%d_%d\""; + String findValueFormat = "/response/docs/[%d]/multi_mapping/[%d]/value==\"%s\""; + + List list = new ArrayList<>(); + for (int i = 0; i < required; i++) { + for (int j = 0; j < required; j++) { + list.add(String.format(Locale.ENGLISH, findKeyFormat, i, j, i, j)); + list.add( + String.format( + Locale.ENGLISH, findValueFormat, i, j, mappings.get("key_" + i + "_" + j))); + } + } + String[] tests = list.toArray(new String[0]); + + String response = assertJQ(req("q", "*:*", "indent", "true"), tests); + log.info(response); + } + + @Test + public void testFloatValueXml() { + /* + * key_21.23 + */ + int requiredDocs = 5; + Map mappings = + doAddDocs("float_mapping", requiredDocs, false, NumberType.FLOAT, false); + + String findKeyFormat = "//doc/mapping[@name=\"float_mapping\"]/str[@name=\"key\"][text()='%s']"; + String findValueFormat = "/parent::mapping/float[@name=\"value\"][text()='%s']"; + + String[] tests = + mappings.entrySet().stream() + .map( + e -> + String.format(Locale.ENGLISH, findKeyFormat, e.getKey()) + + String.format(Locale.ENGLISH, findValueFormat, e.getValue())) + .toList() + .toArray(new String[0]); + + String response = assertXmlQ(req("q", "*:*", "indent", "true"), tests); + log.info(response); + } + + @Test + public void testFloatValueJson() throws Exception { + /* + * "float_mapping":{ "key":"key_0", "value":"12.34" } + */ + int requiredDocs = 5; + Map mappings = + doAddDocs("float_mapping", requiredDocs, false, NumberType.FLOAT, true); + + // json output writes everything as str: + String findKeyFormat = "/response/docs/[%d]/float_mapping/key==\"key_%d\""; + String findValueFormat = "/response/docs/[%d]/float_mapping/value==\"%s\""; + + String[] tests = new String[mappings.size() * 2]; + + for (int i = 0, j = 0; i < mappings.size(); i++, j++) { + tests[i] = String.format(Locale.ENGLISH, findKeyFormat, j, j); + tests[i++] = String.format(Locale.ENGLISH, findValueFormat, j, mappings.get("key_" + j)); + } + + String response = assertJQ(req("q", "*:*", "indent", "true", "wt", "json"), tests); + log.info(response); + } + + @Test + public void testDateStrMapping() throws Exception { + /* + * 2025-11-21T16:09:15ZmPfsP + */ + int requiredDocs = 5; + Map mappings = + doAddDocs("date_str_mapping", requiredDocs, false, null, NumberType.DATE, false); + + String findKeyFormat = + "//doc/mapping[@name=\"date_str_mapping\"]/date[@name=\"key\"][text()='%s']"; + String findValueFormat = "/parent::mapping/str[@name=\"value\"][text()='%s']"; + + String[] tests = + mappings.entrySet().stream() + .map( + e -> + String.format(Locale.ENGLISH, findKeyFormat, e.getKey()) + + String.format(Locale.ENGLISH, findValueFormat, e.getValue())) + .toList() + .toArray(new String[0]); + + String response = assertXmlQ(req("q", "*:*", "indent", "true"), tests); + log.info(response); + } + + @Test + public void testSearchField() { + int requiredDocs = 5; + Map mappings = doAddDocs("single_mapping", requiredDocs, false, false); + + String findKeyFormat = + "//doc/mapping[@name=\"single_mapping\"]/str[@name=\"key\"][text()='%s']"; + String findValueFormat = "/parent::mapping/str[@name=\"value\"][text()='%s']"; + + String[] tests = + mappings.entrySet().stream() + .map( + e -> + String.format(Locale.ENGLISH, findKeyFormat, e.getKey()) + + String.format(Locale.ENGLISH, findValueFormat, e.getValue())) + .toList() + .toArray(new String[0]); + + // q=single_mapping:* requires docValues on 'single_mapping' + String response = assertXmlQ(req("q", "single_mapping:*", "indent", "true"), tests); + log.info(response); + } + + // generate string-string mappings + private Map doAddDocs( + String field, int nb, boolean multiVal, boolean predictableStrKey) { + return doAddDocs(field, nb, multiVal, null, null, predictableStrKey); + } + + // generate string-NumberType mappings + private Map doAddDocs( + String field, int nb, boolean multiVal, NumberType subType, boolean predictableStrKey) { + return doAddDocs(field, nb, multiVal, subType, null, predictableStrKey); + } + + // generate solr input documents with the given keyType and subType (value type) + private Map doAddDocs( + String field, + int nb, + boolean multiVal, + NumberType subType, + NumberType keyType, + boolean predictableStrKey) { + List docs = + MappingsTestUtils.generateDocs( + random(), field, nb, multiVal, subType, keyType, predictableStrKey); + Map mappings = new HashMap<>(); + for (SolrInputDocument doc : docs) { + SolrInputField inField = doc.getField(field); + if (inField.getName().endsWith("mapping")) { + Collection values = inField.getValues(); + for (Object value : values) { + String[] mapping = MappingType.parseCommaSeparatedList(value.toString()); + mappings.put(mapping[MappingType.KEY], mapping[MappingType.VALUE]); + } + } + assertU(adoc(doc)); + } + assertU(commit()); + return mappings; + } +} diff --git a/solr/solr-ref-guide/modules/query-guide/pages/mapping-field.adoc b/solr/solr-ref-guide/modules/query-guide/pages/mapping-field.adoc new file mode 100644 index 00000000000..d7b1d443390 --- /dev/null +++ b/solr/solr-ref-guide/modules/query-guide/pages/mapping-field.adoc @@ -0,0 +1,107 @@ += Mapping Field +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +With the *Mapping Field* module you can index key-value pairs as value of a Solr field. + +The module also supports searching for keys or values within the mapping field's key-value pairs. + +== Configuration + +=== Query Parser Plugin + +Mapping Field is a module and therefore its query parser plugin must be configured in `solrconfig.xml`. + +* Enable the `mapping-field` module to make the Mapping Field classes available on Solr's classpath. +See xref:configuration-guide:solr-modules.adoc[Solr Module] for more details. + +* Declaration of the `mappings` query parser. ++ +[source,xml] +---- + +---- + +=== Schema + +==== Field Type Definition + +The `solr.MappingType` field type requires defining only one of `subFieldSuffix` or `subFieldType`, to set the type of the `value` in the mapping. The `key` field type defaults to the first defined `solr.StrField`. + +* `subFieldSuffix`: One of the `dynamicField` suffixes available in the schema. This property defines the field type of the `value` in the mapping. Ignored if `subFieldType`. is provided. +* `subFieldType`: Name of one of the `fieldType` available in the schema. This property defines the field type of the `value` in the mapping. Optional if `subFieldSuffix` is provided. +* `keyFieldSuffix`: One of the `dynamicField` suffixes available in the schema. This property defines the field type of the `key` in the mapping. Ignored if `keyFieldType`. is provided. +* `keyFieldType`: Name of one of the `fieldType` available in the schema. This property defines the field type of the `key` in the mapping. Optional if `keyFieldSuffix` is provided. + +Examples: +---- + + +---- + +IMPORTANT: Set `docValues=true` on the field type to enable `docValues`. DocValues cannot be enabled on multiValued mapping field definitions. + +[NOTE] +Only primitive field types are supported for keys and values: the numeric and date field types `solr.*PointField`, the boolean field type `solr.BoolField`, and the string field type `solr.StrField`. + +==== Field Definition + +Field of type `solr.MappingType` are defined using Solr fields general properties, except `docValues=true`, which must be enabled only on the field type. + +[source,xml] +---- + + +---- + +== Usage + +=== Indexing + +In a Solr input document, the value of a mapping field must be a CVS representing the key-value mapping. Only one CSV key-value mapping is supported per input field. + +---- +"key 1","value 1" +"key 1","12.34" +---- + +=== Searching + +The field type `solr.MappingType` support existence query using the mapping field name. + +[source,text] +http://localhost:8983/solr/my_collection/select?q=str_fl_mapping:* + +*Query Parser Plugin* + +The query parser `MappingsQParserPlugin` supports searching for values of the keys and/or values of a given mapping field, or across all mappings. + +[source,text] +http://localhost:8983/solr/my_collection/select?q={!mappings f=str_fl_mapping key="key 1" value="[10.0 TO *]"} + +_*Local Parameters*_ + +See xref:query-guide:local-params.adoc[Local Params] for general usage of local parameters. + +* `type=mappings`, or `!mappings`: Enables the `MappingsQParserPlugin`. +* `f`: Optional. Name of a mapping field. If omitted, all mapping fields will be searched. +* `key`: Optional. Value of a mapping key to search for. May contain wildcards `*`or `?`. May be a range query if the mapping key field type supports range queries. +* `value`: Optional. Value of a mapping value to search for. May contain wildcards `*`or `?`. May be a range query if the mapping value field type supports range queries. + +[NOTE] +IMPORTANT: A range query used in the local parameters *must* be URL-encoded OR wrapped in quotes. + diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java index 8068705858a..05051dff53a 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java @@ -881,13 +881,21 @@ private static void checkUpdateU(String message, String update, boolean shouldSu } } - /** Validates a query matches some XPath test expressions and closes the query */ - public static void assertQ(SolrQueryRequest req, String... tests) { - assertQ(null, req, tests); + /** + * Validates a query matches some XPath test expressions and closes the query. + * + * @return the response as XML string + */ + public static String assertXmlQ(SolrQueryRequest req, String... tests) { + return assertXmlQ(null, req, tests); } - /** Validates a query matches some XPath test expressions and closes the query */ - public static void assertQ(String message, SolrQueryRequest req, String... tests) { + /** + * Validates a query matches some XPath test expressions and closes the query. + * + * @return the response as XML string + */ + public static String assertXmlQ(String message, SolrQueryRequest req, String... tests) { try { String m = (null == message) ? "" : message + " "; // TODO log 'm' !!! // since the default (standard) response format is now JSON @@ -921,6 +929,7 @@ public static void assertQ(String message, SolrQueryRequest req, String... tests fail(msg); } + return response; } catch (XPathExpressionException e1) { throw new RuntimeException("XPath is invalid", e1); } catch (Exception e2) { @@ -929,6 +938,16 @@ public static void assertQ(String message, SolrQueryRequest req, String... tests } } + /** Validates a query matches some XPath test expressions and closes the query */ + public static void assertQ(SolrQueryRequest req, String... tests) { + assertQ(null, req, tests); + } + + /** Validates a query matches some XPath test expressions and closes the query */ + public static void assertQ(String message, SolrQueryRequest req, String... tests) { + assertXmlQ(message, req, tests); + } + /** Makes a query request and returns the JSON string response */ public static String JQ(SolrQueryRequest req) throws Exception { SolrParams params = req.getParams();