Skip to content

Commit d6d2fa8

Browse files
Kubik42Kubik42
authored
Improve block loader for source only runtime IP fields (#135393)
* Improve block loader for source only runtime IP fields * Update docs/changelog/135393.yaml * Extracted ip block reader into a separate class
1 parent 77b4fd5 commit d6d2fa8

File tree

5 files changed

+219
-40
lines changed

5 files changed

+219
-40
lines changed

‎docs/changelog/135393.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 135393
2+
summary: Improve block loader for source only runtime IP fields
3+
area: Mapping
4+
type: enhancement
5+
issues: []

‎server/src/main/java/org/elasticsearch/index/mapper/IpFallbackSyntheticSourceReader.java‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper;
11+
12+
import org.apache.lucene.document.InetAddressPoint;
13+
import org.apache.lucene.util.BytesRef;
14+
import org.elasticsearch.common.network.InetAddresses;
15+
import org.elasticsearch.xcontent.XContentParser;
16+
17+
import java.io.IOException;
18+
import java.net.InetAddress;
19+
import java.util.List;
20+
21+
public class IpFallbackSyntheticSourceReader extends FallbackSyntheticSourceBlockLoader.SingleValueReader<InetAddress> {
22+
23+
public IpFallbackSyntheticSourceReader(Object nullValue) {
24+
super(nullValue);
25+
}
26+
27+
@Override
28+
public void convertValue(Object value, List<InetAddress> accumulator) {
29+
try {
30+
if (value instanceof InetAddress ia) {
31+
accumulator.add(ia);
32+
} else {
33+
InetAddress address = InetAddresses.forString(value.toString());
34+
accumulator.add(address);
35+
}
36+
} catch (Exception e) {
37+
// value is malformed, skip it
38+
}
39+
}
40+
41+
@Override
42+
public void writeToBlock(List<InetAddress> values, BlockLoader.Builder blockBuilder) {
43+
BlockLoader.BytesRefBuilder builder = (BlockLoader.BytesRefBuilder) blockBuilder;
44+
for (InetAddress address : values) {
45+
var bytesRef = new BytesRef(InetAddressPoint.encode(address));
46+
builder.appendBytesRef(bytesRef);
47+
}
48+
}
49+
50+
@Override
51+
protected void parseNonNullValue(XContentParser parser, List<InetAddress> accumulator) throws IOException {
52+
try {
53+
InetAddress address = InetAddresses.forString(parser.text());
54+
accumulator.add(address);
55+
} catch (Exception e) {
56+
// value is malformed, skip it
57+
}
58+
}
59+
60+
}

‎server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java‎

Lines changed: 1 addition & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@
4242
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
4343
import org.elasticsearch.search.lookup.FieldValues;
4444
import org.elasticsearch.search.lookup.SearchLookup;
45-
import org.elasticsearch.xcontent.XContentParser;
4645
import org.elasticsearch.xcontent.XContentString;
4746

4847
import java.io.IOException;
@@ -52,7 +51,6 @@
5251
import java.util.Arrays;
5352
import java.util.Collection;
5453
import java.util.Collections;
55-
import java.util.List;
5654
import java.util.Map;
5755
import java.util.Objects;
5856
import java.util.Set;
@@ -480,44 +478,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
480478
}
481479

482480
private BlockLoader blockLoaderFromFallbackSyntheticSource(BlockLoaderContext blContext) {
483-
var reader = new FallbackSyntheticSourceBlockLoader.SingleValueReader<InetAddress>(nullValue) {
484-
@Override
485-
public void convertValue(Object value, List<InetAddress> accumulator) {
486-
if (value instanceof InetAddress ia) {
487-
accumulator.add(ia);
488-
}
489-
490-
try {
491-
var address = InetAddresses.forString(value.toString());
492-
accumulator.add(address);
493-
} catch (Exception e) {
494-
// Malformed value, skip it
495-
}
496-
}
497-
498-
@Override
499-
protected void parseNonNullValue(XContentParser parser, List<InetAddress> accumulator) throws IOException {
500-
// aligned with #parseCreateField()
501-
String value = parser.text();
502-
503-
try {
504-
var address = InetAddresses.forString(value);
505-
accumulator.add(address);
506-
} catch (Exception e) {
507-
// Malformed value, skip it
508-
}
509-
}
510-
511-
@Override
512-
public void writeToBlock(List<InetAddress> values, BlockLoader.Builder blockBuilder) {
513-
var bytesRefBuilder = (BlockLoader.BytesRefBuilder) blockBuilder;
514-
515-
for (var value : values) {
516-
bytesRefBuilder.appendBytesRef(new BytesRef(InetAddressPoint.encode(value)));
517-
}
518-
}
519-
};
520-
481+
var reader = new IpFallbackSyntheticSourceReader(nullValue);
521482
return new FallbackSyntheticSourceBlockLoader(
522483
reader,
523484
name(),

‎server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,16 @@ private Query cidrQuery(String term, SearchExecutionContext context) {
213213

214214
@Override
215215
public BlockLoader blockLoader(BlockLoaderContext blContext) {
216+
FallbackSyntheticSourceBlockLoader fallbackSyntheticSourceBlockLoader = fallbackSyntheticSourceBlockLoader(
217+
blContext,
218+
BlockLoader.BlockFactory::bytesRefs,
219+
() -> new IpFallbackSyntheticSourceReader(null)
220+
);
221+
222+
if (fallbackSyntheticSourceBlockLoader != null) {
223+
return fallbackSyntheticSourceBlockLoader;
224+
}
216225
return new IpScriptBlockDocValuesReader.IpScriptBlockLoader(leafFactory(blContext.lookup()));
217226
}
227+
218228
}

‎server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.apache.lucene.util.BytesRef;
3131
import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery;
3232
import org.elasticsearch.common.network.InetAddresses;
33+
import org.elasticsearch.common.settings.Settings;
3334
import org.elasticsearch.index.IndexVersion;
3435
import org.elasticsearch.index.fielddata.BinaryScriptFieldData;
3536
import org.elasticsearch.index.fielddata.ScriptDocValues.Strings;
@@ -43,20 +44,27 @@
4344
import org.elasticsearch.script.ScriptType;
4445
import org.elasticsearch.search.DocValueFormat;
4546
import org.elasticsearch.search.MultiValueMode;
47+
import org.elasticsearch.search.lookup.SearchLookup;
4648

4749
import java.io.IOException;
4850
import java.time.ZoneId;
4951
import java.util.ArrayList;
52+
import java.util.Arrays;
5053
import java.util.List;
5154
import java.util.Map;
5255

5356
import static java.util.Collections.emptyMap;
5457
import static org.hamcrest.Matchers.containsInAnyOrder;
5558
import static org.hamcrest.Matchers.equalTo;
59+
import static org.hamcrest.Matchers.instanceOf;
60+
import static org.hamcrest.Matchers.nullValue;
5661
import static org.hamcrest.Matchers.sameInstance;
5762

5863
public class IpScriptFieldTypeTests extends AbstractScriptFieldTypeTestCase {
5964

65+
private static final BytesRef EMPTY_IP = null;
66+
private static final BytesRef MALFORMED_IP = null;
67+
6068
@Override
6169
protected ScriptFactory parseFromSource() {
6270
return IpFieldScript.PARSE_FROM_SOURCE;
@@ -280,6 +288,141 @@ public void testBlockLoader() throws IOException {
280288
}
281289
}
282290

291+
public void testBlockLoaderSourceOnlyRuntimeField() throws IOException {
292+
try (
293+
Directory directory = newDirectory();
294+
RandomIndexWriter iw = new RandomIndexWriter(random(), directory, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE))
295+
) {
296+
// given
297+
// try multiple variations of boolean as they're all encoded slightly differently
298+
iw.addDocuments(
299+
List.of(
300+
List.of(new StoredField("_source", new BytesRef("{\"test\": [\"192.168.0.1\"]}"))),
301+
List.of(new StoredField("_source", new BytesRef("{\"test\": [\"2001:db8::1\"]}"))),
302+
List.of(new StoredField("_source", new BytesRef("{\"test\": [\"\"]}"))),
303+
// ensure a malformed value doesn't crash
304+
List.of(new StoredField("_source", new BytesRef("{\"test\": [\"potato\"]}")))
305+
)
306+
);
307+
IpScriptFieldType fieldType = simpleSourceOnlyMappedFieldType();
308+
List<BytesRef> expected = Arrays.asList(
309+
new BytesRef(InetAddressPoint.encode(InetAddresses.forString("192.168.0.1"))),
310+
new BytesRef(InetAddressPoint.encode(InetAddresses.forString("2001:db8::1"))),
311+
EMPTY_IP,
312+
MALFORMED_IP
313+
);
314+
315+
try (DirectoryReader reader = iw.getReader()) {
316+
// when
317+
BlockLoader loader = fieldType.blockLoader(blContext(Settings.EMPTY, true));
318+
319+
// then
320+
321+
// assert loader is of expected instance type
322+
assertThat(loader, instanceOf(IpScriptBlockDocValuesReader.IpScriptBlockLoader.class));
323+
324+
// ignored source doesn't support column at a time loading:
325+
var columnAtATimeLoader = loader.columnAtATimeReader(reader.leaves().getFirst());
326+
assertThat(columnAtATimeLoader, instanceOf(IpScriptBlockDocValuesReader.class));
327+
328+
var rowStrideReader = loader.rowStrideReader(reader.leaves().getFirst());
329+
assertThat(rowStrideReader, instanceOf(IpScriptBlockDocValuesReader.class));
330+
331+
// assert values
332+
assertThat(blockLoaderReadValuesFromColumnAtATimeReader(reader, fieldType, 0), equalTo(expected));
333+
assertThat(blockLoaderReadValuesFromRowStrideReader(reader, fieldType), equalTo(expected));
334+
}
335+
}
336+
}
337+
338+
public void testBlockLoaderSourceOnlyRuntimeFieldWithSyntheticSource() throws IOException {
339+
try (
340+
Directory directory = newDirectory();
341+
RandomIndexWriter iw = new RandomIndexWriter(random(), directory, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE))
342+
) {
343+
// given
344+
// try multiple variations of boolean as they're all encoded slightly differently
345+
iw.addDocuments(
346+
List.of(
347+
createDocumentWithIgnoredSource("[\"192.168.0.1\"]"),
348+
createDocumentWithIgnoredSource("[\"2001:db8::1\"]"),
349+
createDocumentWithIgnoredSource("[\"\"]"),
350+
// ensure a malformed value doesn't crash
351+
createDocumentWithIgnoredSource("[\"potato\"]")
352+
)
353+
);
354+
355+
Settings settings = Settings.builder().put("index.mapping.source.mode", "synthetic").build();
356+
IpScriptFieldType fieldType = simpleSourceOnlyMappedFieldType();
357+
List<BytesRef> expected = Arrays.asList(
358+
new BytesRef(InetAddressPoint.encode(InetAddresses.forString("192.168.0.1"))),
359+
new BytesRef(InetAddressPoint.encode(InetAddresses.forString("2001:db8::1"))),
360+
EMPTY_IP,
361+
MALFORMED_IP
362+
);
363+
364+
try (DirectoryReader reader = iw.getReader()) {
365+
// when
366+
BlockLoader loader = fieldType.blockLoader(blContext(settings, true));
367+
368+
// then
369+
370+
// assert loader is of expected instance type
371+
assertThat(loader, instanceOf(FallbackSyntheticSourceBlockLoader.class));
372+
373+
// ignored source doesn't support column at a time loading:
374+
var columnAtATimeLoader = loader.columnAtATimeReader(reader.leaves().getFirst());
375+
assertThat(columnAtATimeLoader, nullValue());
376+
377+
var rowStrideReader = loader.rowStrideReader(reader.leaves().getFirst());
378+
assertThat(
379+
rowStrideReader.getClass().getName(),
380+
equalTo("org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader$IgnoredSourceRowStrideReader")
381+
);
382+
383+
// assert values
384+
assertThat(blockLoaderReadValuesFromRowStrideReader(settings, reader, fieldType, true), equalTo(expected));
385+
}
386+
}
387+
}
388+
389+
/**
390+
* Returns a source only mapped field type. This is useful, since the available build() function doesn't override isParsedFromSource()
391+
*/
392+
private IpScriptFieldType simpleSourceOnlyMappedFieldType() {
393+
Script script = new Script(ScriptType.INLINE, "test", "", emptyMap());
394+
IpFieldScript.Factory factory = new IpFieldScript.Factory() {
395+
@Override
396+
public IpFieldScript.LeafFactory newFactory(
397+
String fieldName,
398+
Map<String, Object> params,
399+
SearchLookup searchLookup,
400+
OnScriptError onScriptError
401+
) {
402+
return ctx -> new IpFieldScript(fieldName, params, searchLookup, onScriptError, ctx) {
403+
@Override
404+
@SuppressWarnings("unchecked")
405+
public void execute() {
406+
Map<String, Object> source = (Map<String, Object>) this.getParams().get("_source");
407+
for (Object foo : (List<?>) source.get("test")) {
408+
try {
409+
emit(foo.toString());
410+
} catch (Exception e) {
411+
// skip
412+
}
413+
}
414+
}
415+
};
416+
}
417+
418+
@Override
419+
public boolean isParsedFromSource() {
420+
return true;
421+
}
422+
};
423+
return new IpScriptFieldType("test", factory, script, emptyMap(), OnScriptError.FAIL);
424+
}
425+
283426
@Override
284427
protected Query randomTermsQuery(MappedFieldType ft, SearchExecutionContext ctx) {
285428
return ft.termsQuery(randomList(100, () -> randomIp(randomBoolean())), ctx);

0 commit comments

Comments
 (0)