Skip to content

Commit 82ee0eb

Browse files
bruce-hong-gleanbruce-hong-glean
committed
Starting work, untested
Update tests Tests Tests Cleanup Tested, cleanup Cleanup Signed-off-by: Bruce Hong <[email protected]>
1 parent 52b102e commit 82ee0eb

File tree

8 files changed

+600
-5
lines changed

8 files changed

+600
-5
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
3030
- Add the configurable limit on rule cardinality ([#18663](https://github.com/opensearch-project/OpenSearch/pull/18663))
3131
- [Experimental] Start in "clusterless" mode if a clusterless ClusterPlugin is loaded ([#18479](https://github.com/opensearch-project/OpenSearch/pull/18479))
3232
- [Star-Tree] Add star-tree search related stats ([#18707](https://github.com/opensearch-project/OpenSearch/pull/18707))
33+
- The dynamic mapping parameter supports false_allow_templates ([#18825](https://github.com/opensearch-project/OpenSearch/pull/18825))
3334

3435
### Changed
3536
- Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570))

‎rest-api-spec/src/main/resources/rest-api-spec/test/index/111_false_allow_templates.yml

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
---
2+
"Index documents with setting dynamic parameter to false_allow_templates in the mapping of the index":
3+
- skip:
4+
version: " - 3.11.99"
5+
reason: "introduced in 3.2.0"
6+
7+
- do:
8+
indices.create:
9+
index: test_1
10+
body:
11+
mappings:
12+
dynamic: false_allow_templates
13+
dynamic_templates: [
14+
{
15+
dates: {
16+
"match": "date_*",
17+
"match_mapping_type": "date",
18+
"mapping": {
19+
"type": "date"
20+
}
21+
}
22+
},
23+
{
24+
strings: {
25+
"match": "stringField*",
26+
"match_mapping_type": "string",
27+
"mapping": {
28+
"type": "keyword"
29+
}
30+
}
31+
},
32+
{
33+
object: {
34+
"match": "objectField*",
35+
"match_mapping_type": "object",
36+
"mapping": {
37+
"type": "object",
38+
"properties": {
39+
"bar1": {
40+
"type": "keyword"
41+
},
42+
"bar2": {
43+
"type": "text"
44+
}
45+
}
46+
}
47+
}
48+
}
49+
]
50+
properties:
51+
url:
52+
type: keyword
53+
54+
- do:
55+
index:
56+
index: test_1
57+
id: 1
58+
body: {
59+
url: "https://example.com",
60+
date_timestamp: "2024-06-25T05:11:51.243Z",
61+
stringField: "bar",
62+
objectField: {
63+
bar1: "bar1",
64+
bar2: "bar2"
65+
},
66+
author: "John Doe"
67+
}
68+
69+
- do:
70+
get:
71+
index: test_1
72+
id: 1
73+
- match:
74+
_source:
75+
url: "https://example.com"
76+
date_timestamp: "2024-06-25T05:11:51.243Z"
77+
stringField: "bar"
78+
objectField:
79+
bar1: "bar1"
80+
bar2: "bar2"
81+
82+
- do:
83+
indices.get_mapping:
84+
index: test_1
85+
86+
- match: {test_1.mappings.dynamic: false_allow_templates}
87+
- match: {test_1.mappings.properties.url.type: keyword}
88+
- match: {test_1.mappings.properties.date_timestamp.type: date}
89+
- match: {test_1.mappings.properties.stringField.type: keyword}
90+
- match: {test_1.mappings.properties.objectField.properties.bar1.type: keyword}
91+
- match: {test_1.mappings.properties.objectField.properties.bar2.type: text}

‎rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,3 +190,34 @@ setup:
190190

191191
- match: {test_index1.mappings.dynamic: strict_allow_templates}
192192
- match: {test_index1.mappings.properties.test1.type: text}
193+
194+
---
195+
"post a mapping with setting dynamic to false_allow_templates":
196+
- skip:
197+
version: " - 3.11.99"
198+
reason: "introduced in 3.2.0"
199+
- do:
200+
indices.put_mapping:
201+
index: test_index1
202+
body:
203+
dynamic: false_allow_templates
204+
dynamic_templates: [
205+
{
206+
strings: {
207+
"match": "foo*",
208+
"match_mapping_type": "string",
209+
"mapping": {
210+
"type": "keyword"
211+
}
212+
}
213+
}
214+
]
215+
properties:
216+
test1:
217+
type: text
218+
219+
- do:
220+
indices.get_mapping: {}
221+
222+
- match: {test_index1.mappings.dynamic: false_allow_templates}
223+
- match: {test_index1.mappings.properties.test1.type: text}

‎server/src/main/java/org/opensearch/index/mapper/DocumentParser.java

Lines changed: 117 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -549,12 +549,14 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper,
549549
Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, mapper);
550550
ObjectMapper parentMapper = parentMapperTuple.v2();
551551
ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context);
552+
553+
Mapper.Builder builder = null;
552554
switch (dynamic) {
553555
case STRICT:
554556
throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), mapper.fullPath(), currentFieldName);
555557
case TRUE:
556558
case STRICT_ALLOW_TEMPLATES:
557-
Mapper.Builder builder = findTemplateBuilder(
559+
builder = findTemplateBuilder(
558560
context,
559561
currentFieldName,
560562
XContentFieldType.OBJECT,
@@ -572,6 +574,26 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper,
572574
parseObjectOrField(context, objectMapper);
573575
context.path().remove();
574576
break;
577+
case FALSE_ALLOW_TEMPLATES:
578+
builder = findTemplateBuilder(
579+
context,
580+
currentFieldName,
581+
XContentFieldType.OBJECT,
582+
dynamic,
583+
mapper.fullPath()
584+
);
585+
586+
if (builder == null) {
587+
context.parser().skipChildren();
588+
} else {
589+
Mapper.BuilderContext templateBuilderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path());
590+
objectMapper = builder.build(templateBuilderContext);
591+
context.addDynamicMapper(objectMapper);
592+
context.path().add(currentFieldName);
593+
parseObjectOrField(context, objectMapper);
594+
context.path().remove();
595+
}
596+
break;
575597
case FALSE:
576598
// not dynamic, read everything up to end object
577599
context.parser().skipChildren();
@@ -605,6 +627,7 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper,
605627
Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, parentMapper);
606628
parentMapper = parentMapperTuple.v2();
607629
ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context);
630+
Mapper.Builder builder = null;
608631
switch (dynamic) {
609632
case STRICT:
610633
throw new StrictDynamicMappingException(
@@ -614,7 +637,7 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper,
614637
);
615638
case TRUE:
616639
case STRICT_ALLOW_TEMPLATES:
617-
Mapper.Builder builder = findTemplateBuilder(
640+
builder = findTemplateBuilder(
618641
context,
619642
arrayFieldName,
620643
XContentFieldType.OBJECT,
@@ -640,6 +663,33 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper,
640663
}
641664
}
642665
break;
666+
case FALSE_ALLOW_TEMPLATES:
667+
builder = findTemplateBuilder(
668+
context,
669+
arrayFieldName,
670+
XContentFieldType.OBJECT,
671+
dynamic,
672+
parentMapper.fullPath()
673+
);
674+
if (builder == null) {
675+
context.parser().skipChildren();
676+
} else {
677+
Mapper.BuilderContext templateBuilderContext = new Mapper.BuilderContext(
678+
context.indexSettings().getSettings(),
679+
context.path()
680+
);
681+
mapper = builder.build(templateBuilderContext);
682+
assert mapper != null;
683+
if (parsesArrayValue(mapper)) {
684+
context.addDynamicMapper(mapper);
685+
context.path().add(arrayFieldName);
686+
parseObjectOrField(context, mapper);
687+
context.path().remove();
688+
} else {
689+
parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
690+
}
691+
}
692+
break;
643693
case FALSE:
644694
// TODO: shouldn't this skip, not parse?
645695
parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
@@ -786,12 +836,18 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
786836
if (parseableAsLong && context.root().numericDetection()) {
787837
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG, dynamic, fullPath);
788838
if (builder == null) {
839+
if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) {
840+
return null;
841+
}
789842
builder = newLongBuilder(currentFieldName, context.indexSettings().getSettings());
790843
}
791844
return builder;
792845
} else if (parseableAsDouble && context.root().numericDetection()) {
793846
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE, dynamic, fullPath);
794847
if (builder == null) {
848+
if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) {
849+
return null;
850+
}
795851
builder = newFloatBuilder(currentFieldName, context.indexSettings().getSettings());
796852
}
797853
return builder;
@@ -808,6 +864,9 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
808864
}
809865
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, dateTimeFormatter, dynamic, fullPath);
810866
if (builder == null) {
867+
if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) {
868+
return null;
869+
}
811870
boolean ignoreMalformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings());
812871
builder = new DateFieldMapper.Builder(
813872
currentFieldName,
@@ -824,6 +883,9 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
824883

825884
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING, dynamic, fullPath);
826885
if (builder == null) {
886+
if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) {
887+
return null;
888+
}
827889
builder = new TextFieldMapper.Builder(currentFieldName, context.mapperService().getIndexAnalyzers()).addMultiField(
828890
new KeywordFieldMapper.Builder("keyword").ignoreAbove(256)
829891
);
@@ -836,6 +898,9 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
836898
|| numberType == XContentParser.NumberType.BIG_INTEGER) {
837899
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG, dynamic, fullPath);
838900
if (builder == null) {
901+
if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) {
902+
return null;
903+
}
839904
builder = newLongBuilder(currentFieldName, context.indexSettings().getSettings());
840905
}
841906
return builder;
@@ -844,6 +909,9 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
844909
|| numberType == XContentParser.NumberType.BIG_DECIMAL) {
845910
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE, dynamic, fullPath);
846911
if (builder == null) {
912+
if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) {
913+
return null;
914+
}
847915
// no templates are defined, we use float by default instead of double
848916
// since this is much more space-efficient and should be enough most of
849917
// the time
@@ -854,12 +922,18 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
854922
} else if (token == XContentParser.Token.VALUE_BOOLEAN) {
855923
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.BOOLEAN, dynamic, fullPath);
856924
if (builder == null) {
925+
if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) {
926+
return null;
927+
}
857928
builder = new BooleanFieldMapper.Builder(currentFieldName);
858929
}
859930
return builder;
860931
} else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
861932
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.BINARY, dynamic, fullPath);
862933
if (builder == null) {
934+
if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) {
935+
return null;
936+
}
863937
builder = new BinaryFieldMapper.Builder(currentFieldName);
864938
}
865939
return builder;
@@ -868,6 +942,9 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
868942
if (builder != null) {
869943
return builder;
870944
}
945+
if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) {
946+
return null;
947+
}
871948
}
872949
// TODO how do we identify dynamically that its a binary value?
873950
throw new IllegalStateException(
@@ -882,14 +959,23 @@ private static void parseDynamicValue(
882959
XContentParser.Token token
883960
) throws IOException {
884961
ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context);
962+
885963
if (dynamic == ObjectMapper.Dynamic.STRICT) {
886964
throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), parentMapper.fullPath(), currentFieldName);
887965
}
888966
if (dynamic == ObjectMapper.Dynamic.FALSE) {
889967
return;
890968
}
891-
final Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path());
892969
final Mapper.Builder<?> builder = createBuilderFromDynamicValue(context, token, currentFieldName, dynamic, parentMapper.fullPath());
970+
if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES && builder == null) {
971+
// For FALSE_ALLOW_TEMPLATES, if no template matches, we still need to consume the token
972+
// to maintain proper JSON parsing state
973+
if (token == XContentParser.Token.START_OBJECT || token == XContentParser.Token.START_ARRAY) {
974+
context.parser().skipChildren();
975+
}
976+
return;
977+
}
978+
final Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path());
893979
Mapper mapper = builder.build(builderContext);
894980
context.addDynamicMapper(mapper);
895981

@@ -975,12 +1061,13 @@ private static Tuple<Integer, ObjectMapper> getDynamicParentMapper(
9751061
// One mapping is missing, check if we are allowed to create a dynamic one.
9761062
ObjectMapper.Dynamic dynamic = dynamicOrDefault(parent, context);
9771063

1064+
Mapper.Builder builder = null;
9781065
switch (dynamic) {
9791066
case STRICT:
9801067
throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), parent.fullPath(), paths[i]);
9811068
case STRICT_ALLOW_TEMPLATES:
9821069
case TRUE:
983-
Mapper.Builder builder = findTemplateBuilder(
1070+
builder = findTemplateBuilder(
9841071
context,
9851072
paths[i],
9861073
XContentFieldType.OBJECT,
@@ -1004,6 +1091,31 @@ private static Tuple<Integer, ObjectMapper> getDynamicParentMapper(
10041091
}
10051092
context.addDynamicMapper(mapper);
10061093
break;
1094+
case FALSE_ALLOW_TEMPLATES:
1095+
builder = findTemplateBuilder(
1096+
context,
1097+
paths[i],
1098+
XContentFieldType.OBJECT,
1099+
dynamic,
1100+
parent.fullPath()
1101+
);
1102+
if (builder == null) {
1103+
return new Tuple<>(pathsAdded, parent);
1104+
}
1105+
Mapper.BuilderContext templateBuilderContext = new Mapper.BuilderContext(
1106+
context.indexSettings().getSettings(),
1107+
context.path()
1108+
);
1109+
mapper = (ObjectMapper) builder.build(templateBuilderContext);
1110+
if (mapper.nested() != ObjectMapper.Nested.NO) {
1111+
throw new MapperParsingException(
1112+
"It is forbidden to create dynamic nested objects (["
1113+
+ context.path().pathAsText(paths[i])
1114+
+ "]) through `copy_to` or dots in field names"
1115+
);
1116+
}
1117+
context.addDynamicMapper(mapper);
1118+
break;
10071119
case FALSE:
10081120
// Should not dynamically create any more mappers so return the last mapper
10091121
return new Tuple<>(pathsAdded, parent);
@@ -1079,6 +1191,7 @@ private static Mapper.Builder findTemplateBuilder(
10791191
String fieldFullPath
10801192
) {
10811193
Mapper.Builder builder = context.root().findTemplateBuilder(context, name, matchType);
1194+
10821195
if (builder == null && dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) {
10831196
throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), fieldFullPath, name);
10841197
}

0 commit comments

Comments
 (0)