From 320840ab0b2fcc5d9d3f43b2612a117b1a066d13 Mon Sep 17 00:00:00 2001 From: Sai charan raj Gudala Date: Wed, 28 May 2025 23:48:40 +0530 Subject: [PATCH] SubList function (#5529) * SubList function #5529 Signed-off-by: Sai charan raj Gudala --- .../src/main/antlr/DataPrepperExpression.g4 | 1 + .../expression/ParseTreeCoercionService.java | 5 + .../expression/SubListExpressionFunction.java | 68 +++++++ ...ericExpressionEvaluator_ConditionalIT.java | 10 +- .../ParseTreeCoercionServiceTest.java | 2 +- .../SubListExpressionFunctionTest.java | 174 ++++++++++++++++++ 6 files changed, 252 insertions(+), 8 deletions(-) create mode 100644 data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/SubListExpressionFunction.java create mode 100644 data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/SubListExpressionFunctionTest.java diff --git a/data-prepper-expression/src/main/antlr/DataPrepperExpression.g4 b/data-prepper-expression/src/main/antlr/DataPrepperExpression.g4 index 8a771a4d4b..7031d803cc 100644 --- a/data-prepper-expression/src/main/antlr/DataPrepperExpression.g4 +++ b/data-prepper-expression/src/main/antlr/DataPrepperExpression.g4 @@ -167,6 +167,7 @@ fragment FunctionArg : JsonPointer | String + | SUBTRACT? Integer ; variableIdentifier diff --git a/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java b/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java index 692a8c561a..fc5b5f08c9 100644 --- a/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java +++ b/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java @@ -58,6 +58,11 @@ public Object coercePrimaryTerminalNode(final TerminalNode node, final Event eve } argList.add(trimmedArg); } else { + try { + argList.add(Integer.parseInt(trimmedArg)); + continue; + } catch (final Exception e) { + } throw new RuntimeException("Unsupported type passed as function argument"); } } diff --git a/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/SubListExpressionFunction.java b/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/SubListExpressionFunction.java new file mode 100644 index 0000000000..bc569f8c46 --- /dev/null +++ b/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/SubListExpressionFunction.java @@ -0,0 +1,68 @@ +package org.opensearch.dataprepper.expression; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; + +import org.opensearch.dataprepper.model.event.Event; + +import javax.inject.Named; + +@Named +public class SubListExpressionFunction implements ExpressionFunction { + + @Override + public String getFunctionName() { + return "subList"; + } + + @Override + public Object evaluate(List args, Event event, Function convertLiteralType) { + if (args.size() != 3) { + throw new IllegalArgumentException("subList() takes 3 arguments"); + } + if (!(args.get(0) instanceof String)) { + throw new IllegalArgumentException("subList() takes 1st argument as string type"); + } + int startIndex, endIndex; + try { + if (args.get(1) instanceof Integer) { + startIndex = (Integer) args.get(1); + } else { + String str = (String) args.get(1); + startIndex = Integer.parseInt(str.substring(1, str.length() - 1)); + } + if (args.get(2) instanceof Integer) { + endIndex = (Integer) args.get(2); + } else { + String str = (String) args.get(2); + endIndex = Integer.parseInt(str.substring(1, str.length() - 1)); + } + } catch (NumberFormatException | ClassCastException e) { + throw new IllegalArgumentException("subList() takes 2nd and 3rd arguments as integers"); + } + + String key = (String)args.get(0); + final Object value = event.get(key, Object.class); + if (value == null) return null; + if (!(value instanceof List)) { + throw new RuntimeException(key + " is not of list type"); + } + List sourceList = (List)value; + + if (endIndex == -1) endIndex = sourceList.size(); + + if (startIndex < 0 || startIndex >= sourceList.size()) { + throw new RuntimeException("subList() start index should be between 0 and list length (inclusive)"); + } + + if (endIndex < 0 || endIndex > sourceList.size()) { + throw new RuntimeException("subList() end index should be between 0 and list length or -1 for list length (exclusive)"); + } + if (startIndex > endIndex) { + throw new RuntimeException("subList() start index should be less than or equal to end index"); + } + return new ArrayList<>(sourceList.subList(startIndex, endIndex)); + } + +} diff --git a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenericExpressionEvaluator_ConditionalIT.java b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenericExpressionEvaluator_ConditionalIT.java index f98546b4dd..9100c2d61f 100644 --- a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenericExpressionEvaluator_ConditionalIT.java +++ b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenericExpressionEvaluator_ConditionalIT.java @@ -248,7 +248,8 @@ private static Stream validExpressionArguments() { arguments("/name =~ \".*dataprepper-[0-9]+\"", event("{\"name\": \"dataprepper-abc\"}"), false), arguments("/name =~ \".*dataprepper-[0-9]+\"", event("{\"other\": \"dataprepper-abc\"}"), false), arguments("startsWith(\""+strValue+ UUID.randomUUID() + "\",/status)", event("{\"status\":\""+strValue+"\"}"), true), - arguments("startsWith(\""+ UUID.randomUUID() +strValue+ "\",/status)", event("{\"status\":\""+strValue+"\"}"), false) + arguments("startsWith(\""+ UUID.randomUUID() +strValue+ "\",/status)", event("{\"status\":\""+strValue+"\"}"), false), + arguments("subList(/list, 1, 2) != null", event("{\"list\": [0, 1, 2, 3]}"), true) ); } @@ -320,16 +321,13 @@ private static Stream invalidExpressionSyntaxArguments() { arguments("trueand/status_code", event("{\"status_code\": 200}")), arguments("trueor/status_code", event("{\"status_code\": 200}")), arguments("length(\""+testString+") == "+testStringLength, event("{\"response\": \""+testString+"\"}")), - arguments("hasTags(10)", tagEvent), arguments("hasTags("+ testTag1+")", tagEvent), arguments("hasTags(\""+ testTag1+")", tagEvent), arguments("hasTags(\""+ testTag1+"\","+testTag2+"\")", tagEvent), arguments("hasTags(,\""+testTag2+"\")", tagEvent), arguments("hasTags(\""+testTag2+"\",)", tagEvent), arguments("contains(\""+testTag2+"\",)", tagEvent), - arguments("contains(1234, /strField)", event("{\"intField\":1234,\"strField\":\"string\"}")), arguments("contains(str, /strField)", event("{\"intField\":1234,\"strField\":\"string\"}")), - arguments("contains(/strField, 1234)", event("{\"intField\":1234,\"strField\":\"string\"}")), arguments("/color in {\"blue, \"yellow\", \"green\"}", event("{\"color\": \"yellow\"}")), arguments("/color in {\"blue\", yellow\", \"green\"}", event("{\"color\": \"yellow\"}")), arguments("/color in {\", \"yellow\", \"green\"}", event("{\"color\": \"yellow\"}")), @@ -341,10 +339,8 @@ private static Stream invalidExpressionSyntaxArguments() { arguments("/color in {\"\",blue, \"yellow\", \"green\"}", event("{\"color\": \"yellow\"}")), arguments("/value in {22a2.0, 100}", event("{\"value\": 100}")), arguments("/value in {222, 10a0}", event("{\"value\": 100}")), - arguments("getMetadata(10)", tagEvent), arguments("getMetadata("+ testMetadataKey+ ")", tagEvent), - arguments("getMetadata(\""+ testMetadataKey+")", tagEvent), - arguments("cidrContains(/sourceIp,123)", event("{\"sourceIp\": \"192.0.2.3\"}")) + arguments("getMetadata(\""+ testMetadataKey+")", tagEvent) ); } diff --git a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/ParseTreeCoercionServiceTest.java b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/ParseTreeCoercionServiceTest.java index 22477521be..56e8e0efd8 100644 --- a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/ParseTreeCoercionServiceTest.java +++ b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/ParseTreeCoercionServiceTest.java @@ -293,7 +293,7 @@ void testCoerceTerminalNodeLengthFunctionWithInvalidArgument() { final String value = RandomStringUtils.randomAlphabetic(10); final Event testEvent = createTestEvent(Map.of(key, value)); when(terminalNode.getSymbol()).thenReturn(token); - when(terminalNode.getText()).thenReturn("length(10)"); + when(terminalNode.getText()).thenReturn("length(arg)"); when(expressionFunctionProvider.provideFunction(eq("length"), any(List.class), any(Event.class), any(Function.class))).thenReturn(value.length()); when(token.getType()).thenReturn(DataPrepperExpressionParser.Function); assertThrows(RuntimeException.class, () -> objectUnderTest.coercePrimaryTerminalNode(terminalNode, testEvent)); diff --git a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/SubListExpressionFunctionTest.java b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/SubListExpressionFunctionTest.java new file mode 100644 index 0000000000..d84ec597ce --- /dev/null +++ b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/SubListExpressionFunctionTest.java @@ -0,0 +1,174 @@ +package org.opensearch.dataprepper.expression; + +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.JacksonEvent; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +public class SubListExpressionFunctionTest { + private SubListExpressionFunction subListExpressionFunction; + private Event testEvent; + private Function testFunction; + + private Event createTestEvent(final Object data) { + return JacksonEvent.builder().withEventType("event").withData(data).build(); + } + + + public SubListExpressionFunction createObjectUnderTest() { + testFunction = mock(Function.class); + return new SubListExpressionFunction(); + } + + @Test + void testFunctionName() { + subListExpressionFunction = createObjectUnderTest(); + assertEquals("subList", subListExpressionFunction.getFunctionName()); + } + + @Test + void testWithValidArguments() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4); + testEvent = createTestEvent(Map.of("list", testList)); + assertThat(subListExpressionFunction.evaluate(List.of("/list", "\"1\"", "\"3\""), testEvent, testFunction), equalTo(List.of(2, 3))); + } + + @Test + void testWithValidArgumentsCase2() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4); + testEvent = createTestEvent(Map.of("list", testList)); + assertThat(subListExpressionFunction.evaluate(List.of("/list", "\"1\"", "\"3\""), testEvent, testFunction), equalTo(List.of(2, 3))); + } + + @Test + void testWithValidArgumentsCase3() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4); + testEvent = createTestEvent(Map.of("main", Map.of("list", testList))); + assertThat(subListExpressionFunction.evaluate(List.of("/main/list", "\"1\"", "\"3\""), testEvent, testFunction), equalTo(List.of(2, 3))); + } + + @Test + void testWithValidArgumentsCase4() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4); + testEvent = createTestEvent(Map.of("main", Map.of("list", testList))); + assertThat(subListExpressionFunction.evaluate(List.of("/main/list", "\"1\"", "\"-1\""), testEvent, testFunction), equalTo(List.of(2, 3, 4))); + } + + @Test + void testWithValidArgumentsCase5() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4); + testEvent = createTestEvent(Map.of("list", testList)); + assertThat(subListExpressionFunction.evaluate(List.of("/list", 1, 3), testEvent, testFunction), equalTo(List.of(2, 3))); + } + + @Test + void testWithOutOfBoundArgumentsCase1() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4, 5, 6); + testEvent = createTestEvent(Map.of("list", testList)); + Exception exception = assertThrows(RuntimeException.class, () -> subListExpressionFunction.evaluate(List.of("/list", "\"-1\"", "\"4\""), testEvent, testFunction)); + assertEquals("subList() start index should be between 0 and list length (inclusive)", exception.getMessage()); + } + + @Test + void testWithOutOfBoundArgumentsCase2() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4, 5, 6); + testEvent = createTestEvent(Map.of("list", testList)); + Exception exception = assertThrows(RuntimeException.class, () -> subListExpressionFunction.evaluate(List.of("/list", "\"10\"", "\"4\""), testEvent, testFunction)); + assertEquals("subList() start index should be between 0 and list length (inclusive)", exception.getMessage()); + } + + @Test + void testWithOutOfBoundArgumentsCase3() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4, 5, 6); + testEvent = createTestEvent(Map.of("list", testList)); + Exception exception = assertThrows(RuntimeException.class, () -> subListExpressionFunction.evaluate(List.of("/list", "\"4\"", "\"2\""), testEvent, testFunction)); + assertEquals("subList() start index should be less than or equal to end index", exception.getMessage()); + } + + @Test + void testWithOutOfBoundArgumentsCase4() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4, 5, 6); + testEvent = createTestEvent(Map.of("list", testList)); + Exception exception = assertThrows(RuntimeException.class, () -> subListExpressionFunction.evaluate(List.of("/list", "\"1\"", "\"10\""), testEvent, testFunction)); + assertEquals("subList() end index should be between 0 and list length or -1 for list length (exclusive)", exception.getMessage()); + } + + @Test + void testWithOutOfBoundArgumentsCase5() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4, 5, 6); + testEvent = createTestEvent(Map.of("list", testList)); + Exception exception = assertThrows(RuntimeException.class, () -> subListExpressionFunction.evaluate(List.of("/list", "\"1\"", "\"-2\""), testEvent, testFunction)); + assertEquals("subList() end index should be between 0 and list length or -1 for list length (exclusive)", exception.getMessage()); + } + + @Test + void testWithInvalidArguments() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4, 5, 6); + testEvent = createTestEvent(Map.of("list", testList)); + Exception exception = assertThrows(RuntimeException.class, () -> subListExpressionFunction.evaluate(List.of("/list", "\"5\"", "\"2\""), testEvent, testFunction)); + assertEquals("subList() start index should be less than or equal to end index", exception.getMessage()); + } + + @Test + void testWithInvalidArgumentsCase2() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4, 5, 6); + testEvent = createTestEvent(Map.of("list", testList)); + Exception exception = assertThrows(RuntimeException.class, () -> subListExpressionFunction.evaluate(List.of("/list", "\"five\"", "\"two\""), testEvent, testFunction)); + assertEquals("subList() takes 2nd and 3rd arguments as integers", exception.getMessage()); + } + + @Test + void testWithInvalidArgumentsCase3() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4, 5, 6); + testEvent = createTestEvent(Map.of("list", testList)); + Exception exception = assertThrows(RuntimeException.class, () -> subListExpressionFunction.evaluate(List.of("/list", "\"0\""), testEvent, testFunction)); + assertEquals("subList() takes 3 arguments", exception.getMessage()); + } + + @Test + void testWithInvalidArgumentsCase4() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4, 5, 6); + testEvent = createTestEvent(Map.of("list", testList)); + Exception exception = assertThrows(RuntimeException.class, () -> subListExpressionFunction.evaluate(List.of(1, "\"0\"", "\"2\""), testEvent, testFunction)); + assertEquals("subList() takes 1st argument as string type", exception.getMessage()); + } + + @Test + void testWithInvalidArgumentsCase5() { + subListExpressionFunction = createObjectUnderTest(); + testEvent = createTestEvent(Map.of("list", "testList")); + Exception exception = assertThrows(RuntimeException.class, () -> subListExpressionFunction.evaluate(List.of("/list", "\"0\"", "\"2\""), testEvent, testFunction)); + assertEquals("/list is not of list type", exception.getMessage()); + } + + @Test + void testWithUnknownKeyArgument() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4, 5, 6); + testEvent = createTestEvent(Map.of("list", testList)); + assertThat(subListExpressionFunction.evaluate(List.of("/unknownList", 1, 2), testEvent, testFunction), equalTo(null)); + } +}