Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport 2.x] feat: constant keyword field #13003

Merged
merged 2 commits into from
Apr 22, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
## [Unreleased 2.x]
### Added
- Add explicit dependency to validatePom and generatePom tasks ([#12909](https://github.com/opensearch-project/OpenSearch/pull/12909))
- Constant Keyword Field ([#12285](https://github.com/opensearch-project/OpenSearch/pull/12285))
- [Concurrent Segment Search] Perform buildAggregation concurrently and support Composite Aggregations ([#12697](https://github.com/opensearch-project/OpenSearch/pull/12697))
- Convert ingest processor supports ip type ([#12818](https://github.com/opensearch-project/OpenSearch/pull/12818))
- Allow setting KEYSTORE_PASSWORD through env variable ([#12865](https://github.com/opensearch-project/OpenSearch/pull/12865))
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.index.mapper;

import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.opensearch.OpenSearchParseException;
import org.opensearch.common.annotation.PublicApi;
import org.opensearch.common.regex.Regex;
import org.opensearch.index.fielddata.IndexFieldData;
import org.opensearch.index.fielddata.plain.ConstantIndexFieldData;
import org.opensearch.index.query.QueryShardContext;
import org.opensearch.search.aggregations.support.CoreValuesSourceType;
import org.opensearch.search.lookup.SearchLookup;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;

/**
* Index specific field mapper
*
* @opensearch.api
*/
@PublicApi(since = "2.14.0")
public class ConstantKeywordFieldMapper extends ParametrizedFieldMapper {

public static final String CONTENT_TYPE = "constant_keyword";

private static final String valuePropertyName = "value";

/**
* A {@link Mapper.TypeParser} for the constant keyword field.
*
* @opensearch.internal
*/
public static class TypeParser implements Mapper.TypeParser {
@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
if (!node.containsKey(valuePropertyName)) {
throw new OpenSearchParseException("Field [" + name + "] is missing required parameter [value]");
}
Object value = node.remove(valuePropertyName);
if (!(value instanceof String)) {
throw new OpenSearchParseException("Field [" + name + "] is expected to be a string value");

Check warning on line 54 in server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java#L54

Added line #L54 was not covered by tests
}
return new Builder(name, (String) value);
}
}

private static ConstantKeywordFieldMapper toType(FieldMapper in) {
return (ConstantKeywordFieldMapper) in;
}

/**
* Builder for the binary field mapper
*
* @opensearch.internal
*/
public static class Builder extends ParametrizedFieldMapper.Builder {

private final Parameter<String> value;

public Builder(String name, String value) {
super(name);
this.value = Parameter.stringParam(valuePropertyName, false, m -> toType(m).value, value);
}

@Override
public List<Parameter<?>> getParameters() {
return Arrays.asList(value);
}

@Override
public ConstantKeywordFieldMapper build(BuilderContext context) {
return new ConstantKeywordFieldMapper(
name,
new ConstantKeywordFieldMapper.ConstantKeywordFieldType(buildFullName(context), value.getValue()),
multiFieldsBuilder.build(this, context),
copyTo.build(),
this
);
}
}

/**
* Field type for Index field mapper
*
* @opensearch.internal
*/
@PublicApi(since = "2.14.0")
protected static final class ConstantKeywordFieldType extends ConstantFieldType {

protected final String value;

public ConstantKeywordFieldType(String name, String value) {
super(name, Collections.emptyMap());
this.value = value;
}

@Override
public String typeName() {
return CONTENT_TYPE;
}

@Override
protected boolean matches(String pattern, boolean caseInsensitive, QueryShardContext context) {
return Regex.simpleMatch(pattern, value, caseInsensitive);
}

@Override
public Query existsQuery(QueryShardContext context) {
return new MatchAllDocsQuery();
}

@Override
public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier<SearchLookup> searchLookup) {
return new ConstantIndexFieldData.Builder(fullyQualifiedIndexName, name(), CoreValuesSourceType.BYTES);
}

@Override
public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't " + "support formats.");

Check warning on line 133 in server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java#L133

Added line #L133 was not covered by tests
}

return new SourceValueFetcher(name(), context) {

Check warning on line 136 in server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java#L136

Added line #L136 was not covered by tests
@Override
protected Object parseSourceValue(Object value) {
String keywordValue = value.toString();
return Collections.singletonList(keywordValue);

Check warning on line 140 in server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java#L139-L140

Added lines #L139 - L140 were not covered by tests
}
};
}
}

private final String value;

protected ConstantKeywordFieldMapper(
String simpleName,
MappedFieldType mappedFieldType,
MultiFields multiFields,
CopyTo copyTo,
ConstantKeywordFieldMapper.Builder builder
) {
super(simpleName, mappedFieldType, multiFields, copyTo);
this.value = builder.value.getValue();
}

public ParametrizedFieldMapper.Builder getMergeBuilder() {
return new ConstantKeywordFieldMapper.Builder(simpleName(), this.value).init(this);
}

@Override
protected void parseCreateField(ParseContext context) throws IOException {

final String value;
if (context.externalValueSet()) {
value = context.externalValue().toString();

Check warning on line 168 in server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java#L168

Added line #L168 was not covered by tests
} else {
value = context.parser().textOrNull();
}
if (value == null) {
throw new IllegalArgumentException("constant keyword field [" + name() + "] must have a value");

Check warning on line 173 in server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java#L173

Added line #L173 was not covered by tests
}

if (!value.equals(fieldType().value)) {
throw new IllegalArgumentException("constant keyword field [" + name() + "] must have a value of [" + this.value + "]");
}

}

@Override
public ConstantKeywordFieldMapper.ConstantKeywordFieldType fieldType() {
return (ConstantKeywordFieldMapper.ConstantKeywordFieldType) super.fieldType();
}

@Override
protected String contentType() {
return CONTENT_TYPE;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import org.opensearch.index.mapper.BinaryFieldMapper;
import org.opensearch.index.mapper.BooleanFieldMapper;
import org.opensearch.index.mapper.CompletionFieldMapper;
import org.opensearch.index.mapper.ConstantKeywordFieldMapper;
import org.opensearch.index.mapper.DataStreamFieldMapper;
import org.opensearch.index.mapper.DateFieldMapper;
import org.opensearch.index.mapper.DocCountFieldMapper;
Expand Down Expand Up @@ -168,6 +169,7 @@ public static Map<String, Mapper.TypeParser> getMappers(List<MapperPlugin> mappe
mappers.put(FieldAliasMapper.CONTENT_TYPE, new FieldAliasMapper.TypeParser());
mappers.put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser());
mappers.put(FlatObjectFieldMapper.CONTENT_TYPE, FlatObjectFieldMapper.PARSER);
mappers.put(ConstantKeywordFieldMapper.CONTENT_TYPE, new ConstantKeywordFieldMapper.TypeParser());

for (MapperPlugin mapperPlugin : mapperPlugins) {
for (Map.Entry<String, Mapper.TypeParser> entry : mapperPlugin.getMappers().entrySet()) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.index.mapper;

import org.apache.lucene.index.IndexableField;
import org.opensearch.OpenSearchParseException;
import org.opensearch.common.CheckedConsumer;
import org.opensearch.common.compress.CompressedXContent;
import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.json.JsonXContent;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.index.IndexService;
import org.opensearch.plugins.Plugin;
import org.opensearch.test.InternalSettingsPlugin;
import org.opensearch.test.OpenSearchSingleNodeTestCase;
import org.junit.Before;

import java.io.IOException;
import java.util.Collection;

import static org.hamcrest.Matchers.containsString;

public class ConstantKeywordFieldMapperTests extends OpenSearchSingleNodeTestCase {

private IndexService indexService;
private DocumentMapperParser parser;

@Override
protected Collection<Class<? extends Plugin>> getPlugins() {
return pluginList(InternalSettingsPlugin.class);
}

@Before
public void setup() {
indexService = createIndex("test");
parser = indexService.mapperService().documentMapperParser();
}

public void testDefaultDisabledIndexMapper() throws Exception {

XContentBuilder mapping = XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("field")
.field("type", "constant_keyword")
.field("value", "default_value")
.endObject()
.startObject("field2")
.field("type", "keyword")
.endObject();
mapping = mapping.endObject().endObject().endObject();
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping.toString()));

MapperParsingException e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> {
b.field("field", "sdf");
b.field("field2", "szdfvsddf");
})));
assertThat(
e.getMessage(),
containsString(
"failed to parse field [field] of type [constant_keyword] in document with id '1'. Preview of field's value: 'sdf'"
)
);

final ParsedDocument doc = mapper.parse(source(b -> {
b.field("field", "default_value");
b.field("field2", "field_2_value");
}));

final IndexableField field = doc.rootDoc().getField("field");

// constantKeywordField should not be stored
assertNull(field);
}

public void testMissingDefaultIndexMapper() throws Exception {

final XContentBuilder mapping = XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("field")
.field("type", "constant_keyword")
.endObject()
.startObject("field2")
.field("type", "keyword")
.endObject()
.endObject()
.endObject()
.endObject();

OpenSearchParseException e = expectThrows(
OpenSearchParseException.class,
() -> parser.parse("type", new CompressedXContent(mapping.toString()))
);
assertThat(e.getMessage(), containsString("Field [field] is missing required parameter [value]"));
}

private final SourceToParse source(CheckedConsumer<XContentBuilder, IOException> build) throws IOException {
XContentBuilder builder = JsonXContent.contentBuilder().startObject();
build.accept(builder);
builder.endObject();
return new SourceToParse("test", "1", BytesReference.bytes(builder), MediaTypeRegistry.JSON);
}
}
Loading
Loading