diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java index 669d8452dc0..9ca4eed822d 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java @@ -48,6 +48,13 @@ public class CalcitePlanContext { @Getter @Setter private boolean isResolvingSubquery = false; @Getter @Setter private boolean inCoalesceFunction = false; + /** Fields that are being grouped by in aggregation (for bin operations to preserve originals) */ + @Getter @Setter + private java.util.Set aggregationGroupByFields = new java.util.HashSet<>(); + + /** Total number of group-by fields in current aggregation */ + @Getter @Setter private int aggregationGroupByCount = 0; + /** * The flag used to determine whether we do metadata field projection for user 1. If a project is * never visited, we will do metadata field projection for user 2. Else not because user may @@ -59,6 +66,14 @@ public class CalcitePlanContext { private final Stack correlVar = new Stack<>(); private final Stack> windowPartitions = new Stack<>(); + /** + * Partition columns for bin operations. Used to create partitioned window functions for MIN/MAX + * calculations in WIDTH_BUCKET, matching auto_date_histogram's per-group behavior. Stores + * UnresolvedExpression objects that will be analyzed by bin handlers. + */ + private final Stack> + binPartitionExpressions = new Stack<>(); + @Getter public Map rexLambdaRefMap; private CalcitePlanContext(FrameworkConfig config, SysLimit sysLimit, QueryType queryType) { @@ -134,4 +149,27 @@ public static boolean isLegacyPreferred() { public void putRexLambdaRefMap(Map candidateMap) { this.rexLambdaRefMap.putAll(candidateMap); } + + /** + * Push partition expressions for bin operations. These will be analyzed by bin handlers to create + * PARTITION BY clauses for window functions in WIDTH_BUCKET. + */ + public void pushBinPartitionExpressions( + List partitionExpressions) { + binPartitionExpressions.push(partitionExpressions); + } + + /** Pop partition expressions for bin operations. */ + public void popBinPartitionExpressions() { + if (!binPartitionExpressions.empty()) { + binPartitionExpressions.pop(); + } + } + + /** + * Get current partition expressions for bin operations. Returns empty list if no partitions set. + */ + public List getBinPartitionExpressions() { + return binPartitionExpressions.empty() ? List.of() : binPartitionExpressions.peek(); + } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 06fc02c304d..3bfd65f552b 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -453,6 +453,22 @@ private void validateWildcardPatterns( } } + /** Extract field name from UnresolvedExpression. Handles Field and Alias expressions. */ + private String extractFieldName(UnresolvedExpression expr) { + if (expr instanceof org.opensearch.sql.ast.expression.Field) { + return ((org.opensearch.sql.ast.expression.Field) expr).getField().toString(); + } else if (expr instanceof org.opensearch.sql.ast.expression.Alias) { + org.opensearch.sql.ast.expression.Alias alias = + (org.opensearch.sql.ast.expression.Alias) expr; + if (alias.getDelegated() instanceof org.opensearch.sql.ast.expression.Field) { + return ((org.opensearch.sql.ast.expression.Field) alias.getDelegated()) + .getField() + .toString(); + } + } + return null; + } + private boolean isMetadataField(String fieldName) { return OpenSearchConstants.METADATAFIELD_TYPE_MAP.containsKey(fieldName); } @@ -662,7 +678,26 @@ public RelNode visitBin(Bin node, CalcitePlanContext context) { RexNode binExpression = BinUtils.createBinExpression(node, fieldExpr, context, rexVisitor); String alias = node.getAlias() != null ? node.getAlias() : fieldName; - projectPlusOverriding(List.of(binExpression), List.of(alias), context); + + // Check if this field is used in aggregation grouping with multiple fields + if (context.getAggregationGroupByFields().contains(fieldName) + && context.getAggregationGroupByCount() > 1) { + // For multi-field aggregation: preserve BOTH original field and binned field + // The binned field (fieldName_bin) is used for grouping + // The original field is used for MIN aggregation to show actual timestamps per group + List allFields = new ArrayList<>(context.relBuilder.fields()); + List allFieldNames = + new ArrayList<>(context.relBuilder.peek().getRowType().getFieldNames()); + + // Add the binned field with _bin suffix + allFields.add(binExpression); + allFieldNames.add(fieldName + "_bin"); + + context.relBuilder.project(allFields, allFieldNames); + } else { + // For non-aggregation queries OR single-field binning: replace field with binned value + projectPlusOverriding(List.of(binExpression), List.of(alias), context); + } return context.relBuilder.peek(); } @@ -1020,20 +1055,98 @@ private Pair, List> resolveAttributesForAggregation( CalcitePlanContext context) { List aggCallList = aggExprList.stream().map(expr -> aggVisitor.analyze(expr, context)).toList(); - List groupByList = - groupExprList.stream().map(expr -> rexVisitor.analyze(expr, context)).toList(); - return Pair.of(groupByList, aggCallList); + + // Get available field names in the current relation + List availableFields = context.relBuilder.peek().getRowType().getFieldNames(); + + // Build group-by list, replacing fields with their _bin columns if they exist + List groupByList = new ArrayList<>(); + List additionalAggCalls = new ArrayList<>(); + + // Track if we have bin columns - we'll need this to decide whether to add MIN aggregations + boolean hasBinColumns = false; + int nonBinGroupByCount = 0; + + for (UnresolvedExpression groupExpr : groupExprList) { + RexNode resolvedExpr = rexVisitor.analyze(groupExpr, context); + + // Extract field name from UnresolvedExpression + String fieldName = extractFieldName(groupExpr); + + // Check if this field has a corresponding _bin column + if (fieldName != null) { + String binColumnName = fieldName + "_bin"; + if (availableFields.contains(binColumnName)) { + // Use the _bin column for grouping + groupByList.add(context.relBuilder.field(binColumnName)); + hasBinColumns = true; + continue; + } + } + + // Regular group-by field + groupByList.add(resolvedExpr); + nonBinGroupByCount++; + } + + // Only add MIN aggregations for bin columns if there are OTHER group-by fields + // This matches OpenSearch behavior: + // - With multi-field grouping (e.g., by region, timestamp): Show MIN(timestamp) per group + // - With single-field grouping (e.g., by timestamp only): Show bin start time + if (hasBinColumns && nonBinGroupByCount > 0) { + for (UnresolvedExpression groupExpr : groupExprList) { + String fieldName = extractFieldName(groupExpr); + if (fieldName != null) { + String binColumnName = fieldName + "_bin"; + if (availableFields.contains(binColumnName)) { + // Add MIN(original_field) to show minimum timestamp per bin + additionalAggCalls.add( + context.relBuilder.min(context.relBuilder.field(fieldName)).as(fieldName)); + } + } + } + } + + // Combine original aggregations with additional MIN aggregations for binned fields + List combinedAggCalls = new ArrayList<>(aggCallList); + combinedAggCalls.addAll(additionalAggCalls); + + return Pair.of(groupByList, combinedAggCalls); } @Override public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) { - visitChildren(node, context); - + // Prepare partition columns for bin operations before visiting children + // This allows WIDTH_BUCKET to use per-group min/max (matching auto_date_histogram) List aggExprList = node.getAggExprList(); List groupExprList = new ArrayList<>(); + UnresolvedExpression span = node.getSpan(); + if (Objects.nonNull(span)) { + groupExprList.add(span); + } + groupExprList.addAll(node.getGroupExprList()); + + // Store group-by field names and count so bin operations can preserve original fields + java.util.Set savedGroupByFields = context.getAggregationGroupByFields(); + int savedGroupByCount = context.getAggregationGroupByCount(); + context.setAggregationGroupByFields(new java.util.HashSet<>()); + context.setAggregationGroupByCount(groupExprList.size()); + for (UnresolvedExpression groupExpr : groupExprList) { + String fieldName = extractFieldName(groupExpr); + if (fieldName != null) { + context.getAggregationGroupByFields().add(fieldName); + } + } + + visitChildren(node, context); + + // Restore previous group-by fields and count + context.setAggregationGroupByFields(savedGroupByFields); + context.setAggregationGroupByCount(savedGroupByCount); + + groupExprList.clear(); // The span column is always the first column in result whatever // the order of span in query is first or last one - UnresolvedExpression span = node.getSpan(); if (Objects.nonNull(span)) { groupExprList.add(span); List timeSpanFilters = @@ -1093,23 +1206,86 @@ public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) { // the sequence of output schema is "count, colA, colB". List outputFields = context.relBuilder.fields(); int numOfOutputFields = outputFields.size(); - int numOfAggList = aggExprList.size(); + int numOfUserAggregations = aggExprList.size(); List reordered = new ArrayList<>(numOfOutputFields); - // Add aggregation results first - List aggRexList = - outputFields.subList(numOfOutputFields - numOfAggList, numOfOutputFields); - reordered.addAll(aggRexList); - // Add group by columns - List aliasedGroupByList = - aggregationAttributes.getLeft().stream() - .map(this::extractAliasLiteral) - .flatMap(Optional::stream) - .map(ref -> ((RexLiteral) ref).getValueAs(String.class)) - .map(context.relBuilder::field) - .map(f -> (RexNode) f) - .toList(); - reordered.addAll(aliasedGroupByList); - context.relBuilder.project(reordered); + + // Add user-specified aggregation results first (exclude MIN aggregations for binned fields) + List userAggRexList = + outputFields.subList( + numOfOutputFields - aggregationAttributes.getRight().size(), + numOfOutputFields - aggregationAttributes.getRight().size() + numOfUserAggregations); + + // Wrap AVG aggregations with ROUND to fix floating point precision + for (int i = 0; i < userAggRexList.size(); i++) { + RexNode aggRex = userAggRexList.get(i); + UnresolvedExpression aggExpr = aggExprList.get(i); + + // Unwrap Alias to get to the actual aggregation function + UnresolvedExpression actualAggExpr = aggExpr; + if (aggExpr instanceof org.opensearch.sql.ast.expression.Alias) { + actualAggExpr = ((org.opensearch.sql.ast.expression.Alias) aggExpr).getDelegated(); + } + + // Check if this is an AVG aggregation + if (actualAggExpr instanceof org.opensearch.sql.ast.expression.AggregateFunction) { + org.opensearch.sql.ast.expression.AggregateFunction aggFunc = + (org.opensearch.sql.ast.expression.AggregateFunction) actualAggExpr; + if ("avg".equalsIgnoreCase(aggFunc.getFuncName())) { + // Wrap with ROUND(value, 2) + aggRex = + context.relBuilder.call( + org.apache.calcite.sql.fun.SqlStdOperatorTable.ROUND, + aggRex, + context.rexBuilder.makeLiteral( + 2, + context + .relBuilder + .getTypeFactory() + .createSqlType(org.apache.calcite.sql.type.SqlTypeName.INTEGER), + false)); + } + } + reordered.add(aggRex); + } + + // Add group by columns, replacing _bin columns with their MIN aggregations + // Get field names from the aggregate output (group-by fields come first) + List allFieldNames = context.relBuilder.peek().getRowType().getFieldNames(); + int numGroupByFields = aggregationAttributes.getLeft().size(); + + List outputFieldNames = new ArrayList<>(); + + for (int i = 0; i < numGroupByFields; i++) { + String fieldName = allFieldNames.get(i); + if (fieldName.endsWith("_bin")) { + // This is a bin column + String originalFieldName = fieldName.substring(0, fieldName.length() - 4); // Remove "_bin" + // Check if we have a MIN aggregation for this field (only present for multi-field grouping) + if (allFieldNames.contains(originalFieldName)) { + // Use the MIN aggregation + reordered.add(context.relBuilder.field(originalFieldName)); + outputFieldNames.add(originalFieldName); + } else { + // Use the bin column directly (for single-field binning) - rename to original name + reordered.add(context.relBuilder.field(fieldName)); + outputFieldNames.add(originalFieldName); // Rename _bin field to original name + } + } else { + // Regular group-by field + reordered.add(context.relBuilder.field(fieldName)); + outputFieldNames.add(fieldName); + } + } + + // Add aggregation field names (after group-by fields in the reordered list) + // The user aggregations are at the beginning of reordered list, so we add their names + int aggStartIndex = numOfOutputFields - aggregationAttributes.getRight().size(); + for (int i = aggStartIndex; i < aggStartIndex + numOfUserAggregations; i++) { + outputFieldNames.add( + 0, allFieldNames.get(i)); // Add at beginning to match reordered list order + } + + context.relBuilder.project(reordered, outputFieldNames); return context.relBuilder.peek(); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/CountBinHandler.java b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/CountBinHandler.java index 9716eab993c..8daedb94673 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/CountBinHandler.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/CountBinHandler.java @@ -6,7 +6,6 @@ package org.opensearch.sql.calcite.utils.binning.handlers; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.tree.Bin; import org.opensearch.sql.ast.tree.CountBin; @@ -30,20 +29,21 @@ public RexNode createExpression( requestedBins = BinConstants.DEFAULT_BINS; } - // Calculate data range using window functions + // Calculate global MIN and MAX using window functions RexNode minValue = context.relBuilder.min(fieldExpr).over().toRex(); RexNode maxValue = context.relBuilder.max(fieldExpr).over().toRex(); - RexNode dataRange = context.relBuilder.call(SqlStdOperatorTable.MINUS, maxValue, minValue); // Convert start/end parameters RexNode startValue = convertParameter(countBin.getStart(), context); RexNode endValue = convertParameter(countBin.getEnd(), context); - // WIDTH_BUCKET(field_value, num_bins, data_range, max_value) + // WIDTH_BUCKET(field_value, num_bins, min_value, max_value) + // Note: We pass minValue instead of dataRange - WIDTH_BUCKET will calculate the range + // internally RexNode numBins = context.relBuilder.literal(requestedBins); return context.rexBuilder.makeCall( - PPLBuiltinOperators.WIDTH_BUCKET, fieldExpr, numBins, dataRange, maxValue); + PPLBuiltinOperators.WIDTH_BUCKET, fieldExpr, numBins, minValue, maxValue); } private RexNode convertParameter( diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/MinSpanBinHandler.java b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/MinSpanBinHandler.java index 31e3c11d243..0f28eb7a6f5 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/MinSpanBinHandler.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/MinSpanBinHandler.java @@ -36,7 +36,7 @@ public RexNode createExpression( Number minspanNum = (Number) ((RexLiteral) minspanValue).getValue(); double minspan = minspanNum.doubleValue(); - // Calculate data range using window functions + // Calculate global data range using window functions RexNode minValue = context.relBuilder.min(fieldExpr).over().toRex(); RexNode maxValue = context.relBuilder.max(fieldExpr).over().toRex(); RexNode dataRange = context.relBuilder.call(SqlStdOperatorTable.MINUS, maxValue, minValue); diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/RangeBinHandler.java b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/RangeBinHandler.java index 85e2b701528..e00c980b3bf 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/RangeBinHandler.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/handlers/RangeBinHandler.java @@ -22,7 +22,7 @@ public RexNode createExpression( RangeBin rangeBin = (RangeBin) node; - // Simple MIN/MAX calculation - cleaner than complex CASE expressions + // Simple global MIN/MAX calculation - cleaner than complex CASE expressions RexNode dataMin = context.relBuilder.min(fieldExpr).over().toRex(); RexNode dataMax = context.relBuilder.max(fieldExpr).over().toRex(); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/WidthBucketFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/WidthBucketFunction.java index 160827c7961..79503e37316 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/WidthBucketFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/binning/WidthBucketFunction.java @@ -5,6 +5,9 @@ package org.opensearch.sql.expression.function.udf.binning; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; @@ -22,6 +25,7 @@ import org.opensearch.sql.calcite.utils.binning.BinConstants; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; +import org.opensearch.sql.utils.DateTimeFormatters; /** * WIDTH_BUCKET(field_value, num_bins, data_range, max_value) - Histogram bucketing function. @@ -76,35 +80,84 @@ public Expression implement( RexToLixTranslator translator, RexCall call, List translatedOperands) { Expression fieldValue = translatedOperands.get(0); Expression numBins = translatedOperands.get(1); - Expression dataRange = translatedOperands.get(2); + Expression minValue = translatedOperands.get(2); Expression maxValue = translatedOperands.get(3); + // Pass the field type information to help detect timestamps + RelDataType fieldType = call.getOperands().get(0).getType(); + boolean isTimestampField = dateRelatedType(fieldType); + Expression isTimestamp = Expressions.constant(isTimestampField); + + // For timestamp fields, keep as-is (don't convert to Number) + // For numeric fields, convert to Number + Expression fieldValueExpr = + isTimestampField ? fieldValue : Expressions.convert_(fieldValue, Number.class); + Expression minValueExpr = + isTimestampField ? minValue : Expressions.convert_(minValue, Number.class); + Expression maxValueExpr = + isTimestampField ? maxValue : Expressions.convert_(maxValue, Number.class); + return Expressions.call( WidthBucketImplementor.class, "calculateWidthBucket", - Expressions.convert_(fieldValue, Number.class), + fieldValueExpr, Expressions.convert_(numBins, Number.class), - Expressions.convert_(dataRange, Number.class), - Expressions.convert_(maxValue, Number.class)); + minValueExpr, + maxValueExpr, + isTimestamp); } - /** Width bucket calculation using nice number algorithm. */ + /** Width bucket calculation dispatcher - delegates to timestamp or numeric method. */ public static String calculateWidthBucket( - Number fieldValue, Number numBinsParam, Number dataRange, Number maxValue) { - if (fieldValue == null || numBinsParam == null || dataRange == null || maxValue == null) { + Object fieldValue, + Number numBinsParam, + Object minValue, + Object maxValue, + boolean isTimestamp) { + if (fieldValue == null || numBinsParam == null || minValue == null || maxValue == null) { return null; } - double value = fieldValue.doubleValue(); int numBins = numBinsParam.intValue(); - if (numBins < BinConstants.MIN_BINS || numBins > BinConstants.MAX_BINS) { return null; } - double range = dataRange.doubleValue(); - double max = maxValue.doubleValue(); + return isTimestamp + ? calculateTimestampWidthBucket(fieldValue, numBins, minValue, maxValue) + : calculateNumericWidthBucket(fieldValue, numBins, minValue, maxValue); + } + + /** Width bucket calculation for timestamp fields. */ + private static String calculateTimestampWidthBucket( + Object fieldValue, int numBins, Object minValue, Object maxValue) { + // Convert all timestamp values to milliseconds + long fieldMillis = convertTimestampToMillis(fieldValue); + long minMillis = convertTimestampToMillis(minValue); + long maxMillis = convertTimestampToMillis(maxValue); + + // Calculate range + long rangeMillis = maxMillis - minMillis; + if (rangeMillis <= 0) { + return null; + } + + return calculateTimestampBucket(fieldMillis, numBins, rangeMillis, minMillis); + } + + /** Width bucket calculation for numeric fields using nice number algorithm. */ + private static String calculateNumericWidthBucket( + Object fieldValue, int numBins, Object minValue, Object maxValue) { + Number numericValue = (Number) fieldValue; + Number numericMin = (Number) minValue; + Number numericMax = (Number) maxValue; + double value = numericValue.doubleValue(); + double min = numericMin.doubleValue(); + double max = numericMax.doubleValue(); + + // Calculate range + double range = max - min; if (range <= 0) { return null; } @@ -152,6 +205,94 @@ private static double calculateOptimalWidth( return optimalWidth; } + /** + * Convert timestamp value to milliseconds. Handles both numeric (Long) milliseconds and String + * formatted timestamps. Supports multiple formats: yyyy-MM-dd HH:mm:ss[.SSSSSSSSS], yyyy-MM-dd + * HH:mm, yyyy-MM-dd. + */ + private static long convertTimestampToMillis(Object timestamp) { + if (timestamp instanceof Number) { + return ((Number) timestamp).longValue(); + } else if (timestamp instanceof String) { + // Parse timestamp string using flexible date-time formatter + // Use LocalDateTime to parse without timezone, then convert to UTC + String timestampStr = (String) timestamp; + java.time.LocalDateTime localDateTime = + java.time.LocalDateTime.parse( + timestampStr, DateTimeFormatters.DATE_TIMESTAMP_FORMATTER); + // Assume the timestamp is in UTC and convert to epoch millis + return localDateTime.atZone(ZoneOffset.UTC).toInstant().toEpochMilli(); + } else { + throw new IllegalArgumentException("Unsupported timestamp type: " + timestamp.getClass()); + } + } + + /** + * Calculate timestamp bucket using auto_date_histogram interval selection. Timestamps are in + * milliseconds since epoch. Bins are aligned to the minimum timestamp, not to calendar + * boundaries. + */ + private static String calculateTimestampBucket( + long timestampMillis, int numBins, long rangeMillis, long minMillis) { + // Calculate target width in milliseconds + long targetWidthMillis = rangeMillis / numBins; + + // Select appropriate time interval (same as OpenSearch auto_date_histogram) + long intervalMillis = selectTimeInterval(targetWidthMillis); + + // Floor timestamp to the interval boundary aligned with minMillis + // This ensures bins start at the data's minimum value, like OpenSearch auto_date_histogram + long offsetFromMin = timestampMillis - minMillis; + long intervalsSinceMin = offsetFromMin / intervalMillis; + long binStartMillis = minMillis + (intervalsSinceMin * intervalMillis); + + // Format as ISO 8601 timestamp string + return formatTimestamp(binStartMillis); + } + + /** + * Select the appropriate time interval based on target width. Uses the same intervals as + * OpenSearch auto_date_histogram: 1s, 5s, 10s, 30s, 1m, 5m, 10m, 30m, 1h, 3h, 12h, 1d, 7d, 1M, + * 1y + */ + private static long selectTimeInterval(long targetWidthMillis) { + // Define nice time intervals in milliseconds + long[] intervals = { + 1000L, // 1 second + 5000L, // 5 seconds + 10000L, // 10 seconds + 30000L, // 30 seconds + 60000L, // 1 minute + 300000L, // 5 minutes + 600000L, // 10 minutes + 1800000L, // 30 minutes + 3600000L, // 1 hour + 10800000L, // 3 hours + 43200000L, // 12 hours + 86400000L, // 1 day + 604800000L, // 7 days + 2592000000L, // 30 days (approximate month) + 31536000000L // 365 days (approximate year) + }; + + // Find the smallest interval that is >= target width + for (long interval : intervals) { + if (interval >= targetWidthMillis) { + return interval; + } + } + + // If target is larger than all intervals, use the largest + return intervals[intervals.length - 1]; + } + + /** Format timestamp in milliseconds as SQL literal string (yyyy-MM-dd HH:mm:ss). */ + private static String formatTimestamp(long timestampMillis) { + Instant instant = Instant.ofEpochMilli(timestampMillis); + ZonedDateTime zdt = instant.atZone(ZoneOffset.UTC); + return zdt.format(DateTimeFormatters.SQL_LITERAL_DATE_TIME_FORMAT); + } + /** Format range string with appropriate precision. */ private static String formatRange(double binStart, double binEnd, double span) { if (isIntegerSpan(span) && isIntegerValue(binStart) && isIntegerValue(binEnd)) { diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java index 13e6b4a47e1..7545b453f7c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java @@ -27,6 +27,7 @@ public void init() throws Exception { loadIndex(Index.BANK); loadIndex(Index.EVENTS_NULL); loadIndex(Index.TIME_TEST_DATA); + loadIndex(Index.TIME_BINS_TEST_DATA); } @Test @@ -862,8 +863,6 @@ public void testBinFloatingPointSpanWithStatsCount() throws IOException { @Test public void testStatsWithBinsOnTimeField_Count() throws IOException { - // TODO: Remove this after addressing https://github.com/opensearch-project/sql/issues/4317 - enabledOnlyWhenPushdownIsEnabled(); JSONObject result = executeQuery("source=events_null | bin @timestamp bins=3 | stats count() by @timestamp"); @@ -900,8 +899,6 @@ public void testStatsWithBinsOnTimeField_Count() throws IOException { @Test public void testStatsWithBinsOnTimeField_Avg() throws IOException { - // TODO: Remove this after addressing https://github.com/opensearch-project/sql/issues/4317 - enabledOnlyWhenPushdownIsEnabled(); JSONObject result = executeQuery( @@ -941,8 +938,6 @@ public void testStatsWithBinsOnTimeField_Avg() throws IOException { @Test public void testStatsWithBinsOnTimeAndTermField_Count() throws IOException { - // TODO: Remove this after addressing https://github.com/opensearch-project/sql/issues/4317 - enabledOnlyWhenPushdownIsEnabled(); JSONObject result = executeQuery( @@ -964,8 +959,6 @@ public void testStatsWithBinsOnTimeAndTermField_Count() throws IOException { @Test public void testStatsWithBinsOnTimeAndTermField_Avg() throws IOException { - // TODO: Remove this after addressing https://github.com/opensearch-project/sql/issues/4317 - enabledOnlyWhenPushdownIsEnabled(); JSONObject result = executeQuery( @@ -984,4 +977,285 @@ public void testStatsWithBinsOnTimeAndTermField_Avg() throws IOException { rows(50, "us-east", "2024-07-01 00:05:00"), rows(40.25, "us-west", "2024-07-01 00:01:00")); } + + @Test + public void testBinTimestampBins20WithStats() throws IOException { + + JSONObject result = + executeQuery( + "source=opensearch-sql_test_index_time_bins_data | where @timestamp < '2025-07-28" + + " 10:06:00' | bin @timestamp bins=20 | stats count() by @timestamp | sort" + + " @timestamp"); + + verifySchema( + result, schema("count()", null, "bigint"), schema("@timestamp", null, "timestamp")); + + verifyDataRows( + result, + rows(3, "2025-07-28 10:00:00"), // Records at: 10:00:00, 10:00:17, 10:00:23 + rows(2, "2025-07-28 10:00:30"), // Records at: 10:00:41, 10:00:58 + rows(2, "2025-07-28 10:01:00"), // Records at: 10:01:12, 10:01:29 + rows(1, "2025-07-28 10:01:30"), // Records at: 10:01:47 + rows(2, "2025-07-28 10:02:00"), // Records at: 10:02:03, 10:02:21 + rows(2, "2025-07-28 10:02:30"), // Records at: 10:02:38, 10:02:55 + rows(2, "2025-07-28 10:03:00"), // Records at: 10:03:09, 10:03:26 + rows(1, "2025-07-28 10:03:30"), // Records at: 10:03:44 + rows(2, "2025-07-28 10:04:00"), // Records at: 10:04:01, 10:04:18 + rows(2, "2025-07-28 10:04:30"), // Records at: 10:04:35, 10:04:52 + rows(1, "2025-07-28 10:05:00")); // Records at: 10:05:07 + } + + @Test + public void testBinTimestampBins10WithStats() throws IOException { + + JSONObject result = + executeQuery( + "source=opensearch-sql_test_index_time_bins_data | where @timestamp < '2025-07-28" + + " 10:06:00' | bin @timestamp bins=10 | stats count() by @timestamp | sort" + + " @timestamp"); + + verifySchema( + result, schema("count()", null, "bigint"), schema("@timestamp", null, "timestamp")); + + verifyDataRows( + result, + rows( + 5, + "2025-07-28 10:00:00"), // Records at: 10:00:00, 10:00:17, 10:00:23, 10:00:41, 10:00:58 + rows(3, "2025-07-28 10:01:00"), // Records at: 10:01:12, 10:01:29, 10:01:47 + rows(4, "2025-07-28 10:02:00"), // Records at: 10:02:03, 10:02:21, 10:02:38, 10:02:55 + rows(3, "2025-07-28 10:03:00"), // Records at: 10:03:09, 10:03:26, 10:03:44 + rows(4, "2025-07-28 10:04:00"), // Records at: 10:04:01, 10:04:18, 10:04:35, 10:04:52 + rows(1, "2025-07-28 10:05:00")); // Records at: 10:05:07 + } + + @Test + public void testBinTimestampBins5WithStats() throws IOException { + + JSONObject result = + executeQuery( + "source=opensearch-sql_test_index_time_bins_data | where @timestamp < '2025-07-28" + + " 10:06:00' | bin @timestamp bins=5 | stats count() by @timestamp | sort" + + " @timestamp"); + + verifySchema( + result, schema("count()", null, "bigint"), schema("@timestamp", null, "timestamp")); + + verifyDataRows( + result, + rows(19, "2025-07-28 10:00:00"), // All records from 10:00:00 to 10:04:52 (first 19 records) + rows(1, "2025-07-28 10:05:00")); // Last record at 10:05:07 + } + + @Test + public void testBinTimestampBins10HoursWithStats() throws IOException { + + JSONObject result = + executeQuery( + "source=opensearch-sql_test_index_time_bins_data | bin @timestamp bins=10 | stats" + + " count() by @timestamp | sort @timestamp"); + + verifySchema( + result, schema("count()", null, "bigint"), schema("@timestamp", null, "timestamp")); + + // With 1-hour bins over ~5.4 hours + verifyDataRows( + result, + rows(23, "2025-07-28 10:00:00"), // 10:00:00 to 10:59:59 + rows(4, "2025-07-28 11:00:00"), // 11:00:00 to 11:59:59 + rows(4, "2025-07-28 12:00:00"), // 12:00:00 to 12:59:59 + rows(3, "2025-07-28 13:00:00"), // 13:00:00 to 13:59:59 + rows(4, "2025-07-28 14:00:00"), // 14:00:00 to 14:59:59 + rows(2, "2025-07-28 15:00:00")); // 15:00:00 to 15:59:59 + } + + @Test + public void testBinTimestampBins5HoursWithStats() throws IOException { + + JSONObject result = + executeQuery( + "source=opensearch-sql_test_index_time_bins_data | bin @timestamp bins=5 | stats" + + " count() by @timestamp | sort @timestamp"); + + verifySchema( + result, schema("count()", null, "bigint"), schema("@timestamp", null, "timestamp")); + + // With 3-hour bins over ~5.4 hours + verifyDataRows( + result, + rows(31, "2025-07-28 10:00:00"), // 10:00:00 to 12:59:59 + rows(9, "2025-07-28 13:00:00")); // 13:00:00 to 15:59:59 + } + + @Test + public void testBinTimestampBins3HoursWithStats() throws IOException { + + JSONObject result = + executeQuery( + "source=opensearch-sql_test_index_time_bins_data | bin @timestamp bins=3 | stats" + + " count() by @timestamp | sort @timestamp"); + + verifySchema( + result, schema("count()", null, "bigint"), schema("@timestamp", null, "timestamp")); + + // With 3-hour bins over ~5.4 hours (same as bins=5) + verifyDataRows( + result, + rows(31, "2025-07-28 10:00:00"), // 10:00:00 to 12:59:59 + rows(9, "2025-07-28 13:00:00")); // 13:00:00 to 15:59:59 + } + + @Test + public void testBinTimestampBins20WithoutAggregation() throws IOException { + // Test bins=20 without aggregation (30-second intervals) + // Uses WIDTH_BUCKET UDF, no pushdown to OpenSearch + JSONObject result = + executeQuery( + "source=opensearch-sql_test_index_time_bins_data | where @timestamp < '2025-07-28" + + " 10:06:00' | bin @timestamp bins=20 | fields @timestamp, value | sort @timestamp" + + " | head 10"); + + verifySchema(result, schema("@timestamp", null, "timestamp"), schema("value", null, "int")); + + // Verify 10 rows with 30-second interval bins + verifyDataRows( + result, + rows("2025-07-28 10:00:00", 8945), + rows("2025-07-28 10:00:00", 9012), + rows("2025-07-28 10:00:00", 6712), + rows("2025-07-28 10:00:30", 8917), + rows("2025-07-28 10:00:30", 7162), + rows("2025-07-28 10:01:00", 8429), + rows("2025-07-28 10:01:00", 6985), + rows("2025-07-28 10:01:30", 6583), + rows("2025-07-28 10:02:00", 7823), + rows("2025-07-28 10:02:00", 9156)); + } + + @Test + public void testBinTimestampBins10WithoutAggregation() throws IOException { + // Test bins=10 without aggregation (1-minute intervals) + JSONObject result = + executeQuery( + "source=opensearch-sql_test_index_time_bins_data | where @timestamp < '2025-07-28" + + " 10:06:00' | bin @timestamp bins=10 | fields @timestamp, value | sort @timestamp" + + " | head 10"); + + verifySchema(result, schema("@timestamp", null, "timestamp"), schema("value", null, "int")); + + // Verify 10 rows with 1-minute interval bins + verifyDataRows( + result, + rows("2025-07-28 10:00:00", 8945), + rows("2025-07-28 10:00:00", 9012), + rows("2025-07-28 10:00:00", 6712), + rows("2025-07-28 10:00:00", 8917), + rows("2025-07-28 10:00:00", 7162), + rows("2025-07-28 10:01:00", 8429), + rows("2025-07-28 10:01:00", 6985), + rows("2025-07-28 10:01:00", 6583), + rows("2025-07-28 10:02:00", 7823), + rows("2025-07-28 10:02:00", 9156)); + } + + @Test + public void testBinTimestampBins5WithoutAggregation() throws IOException { + // Test bins=5 without aggregation (5-minute intervals) + JSONObject result = + executeQuery( + "source=opensearch-sql_test_index_time_bins_data | where @timestamp < '2025-07-28" + + " 10:06:00' | bin @timestamp bins=5 | fields @timestamp, value | sort @timestamp" + + " | head 10"); + + verifySchema(result, schema("@timestamp", null, "timestamp"), schema("value", null, "int")); + + // Verify 10 rows with 5-minute interval bins (all in same bin) + verifyDataRows( + result, + rows("2025-07-28 10:00:00", 8945), + rows("2025-07-28 10:00:00", 9012), + rows("2025-07-28 10:00:00", 6712), + rows("2025-07-28 10:00:00", 8917), + rows("2025-07-28 10:00:00", 7162), + rows("2025-07-28 10:00:00", 8429), + rows("2025-07-28 10:00:00", 6985), + rows("2025-07-28 10:00:00", 6583), + rows("2025-07-28 10:00:00", 7823), + rows("2025-07-28 10:00:00", 9156)); + } + + @Test + public void testBinTimestampBins10HoursWithoutAggregation() throws IOException { + // Test bins=10 without aggregation over hour-spanning data (1-hour intervals) + JSONObject result = + executeQuery( + "source=opensearch-sql_test_index_time_bins_data | bin @timestamp bins=10 | fields" + + " @timestamp, value | sort @timestamp | head 10"); + + verifySchema(result, schema("@timestamp", null, "timestamp"), schema("value", null, "int")); + + // Verify 10 rows with 1-hour interval bins (all in same bin) + verifyDataRows( + result, + rows("2025-07-28 10:00:00", 8945), + rows("2025-07-28 10:00:00", 9012), + rows("2025-07-28 10:00:00", 6712), + rows("2025-07-28 10:00:00", 8917), + rows("2025-07-28 10:00:00", 7162), + rows("2025-07-28 10:00:00", 8429), + rows("2025-07-28 10:00:00", 6985), + rows("2025-07-28 10:00:00", 6583), + rows("2025-07-28 10:00:00", 7823), + rows("2025-07-28 10:00:00", 9156)); + } + + @Test + public void testBinTimestampBins5HoursWithoutAggregation() throws IOException { + // Test bins=5 without aggregation over hour-spanning data (3-hour intervals) + JSONObject result = + executeQuery( + "source=opensearch-sql_test_index_time_bins_data | bin @timestamp bins=5 | fields" + + " @timestamp, value | sort @timestamp | head 10"); + + verifySchema(result, schema("@timestamp", null, "timestamp"), schema("value", null, "int")); + + // Verify 10 rows with 3-hour interval bins (all in same bin) + verifyDataRows( + result, + rows("2025-07-28 10:00:00", 8945), + rows("2025-07-28 10:00:00", 9012), + rows("2025-07-28 10:00:00", 6712), + rows("2025-07-28 10:00:00", 8917), + rows("2025-07-28 10:00:00", 7162), + rows("2025-07-28 10:00:00", 8429), + rows("2025-07-28 10:00:00", 6985), + rows("2025-07-28 10:00:00", 6583), + rows("2025-07-28 10:00:00", 7823), + rows("2025-07-28 10:00:00", 9156)); + } + + @Test + public void testBinTimestampBins3HoursWithoutAggregation() throws IOException { + // Test bins=3 without aggregation over hour-spanning data (3-hour intervals) + JSONObject result = + executeQuery( + "source=opensearch-sql_test_index_time_bins_data | bin @timestamp bins=3 | fields" + + " @timestamp, value | sort @timestamp | head 10"); + + verifySchema(result, schema("@timestamp", null, "timestamp"), schema("value", null, "int")); + + // Verify 10 rows with 3-hour interval bins (all in same bin) + verifyDataRows( + result, + rows("2025-07-28 10:00:00", 8945), + rows("2025-07-28 10:00:00", 9012), + rows("2025-07-28 10:00:00", 6712), + rows("2025-07-28 10:00:00", 8917), + rows("2025-07-28 10:00:00", 7162), + rows("2025-07-28 10:00:00", 8429), + rows("2025-07-28 10:00:00", 6985), + rows("2025-07-28 10:00:00", 6583), + rows("2025-07-28 10:00:00", 7823), + rows("2025-07-28 10:00:00", 9156)); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index ad2b0aeda85..3adc3c2f56c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -474,10 +474,10 @@ public void supportPushDownScriptOnTextField() throws IOException { @Test public void testExplainBinWithBins() throws IOException { - String expected = loadExpectedPlan("explain_bin_bins.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_bin_bins.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString("source=opensearch-sql_test_index_account | bin age bins=3 | head 5")); + explainQueryYaml("source=opensearch-sql_test_index_account | bin age bins=3 | head 5")); } @Test @@ -514,22 +514,6 @@ public void testExplainStatsWithSubAggregation() throws IOException { + " @timestamp, region")); } - @Test - public void bucketNullableNotSupportSubAggregation() throws IOException { - // TODO: Don't throw exception after addressing - // https://github.com/opensearch-project/sql/issues/4317 - // When bucketNullable is true, sub aggregation is not supported. Hence we cannot pushdown the - // aggregation in this query. Caused by issue - // https://github.com/opensearch-project/sql/issues/4317, - // bin aggregation on timestamp field won't work if not been push down. - enabledOnlyWhenPushdownIsEnabled(); - assertThrows( - Exception.class, - () -> - explainQueryToString( - "source=events | bin @timestamp bins=3 | stats count() by @timestamp, region")); - } - @Test public void testExplainBinWithSpan() throws IOException { String expected = loadExpectedPlan("explain_bin_span.yaml"); @@ -644,10 +628,10 @@ public void testExplainRegexMatchInEvalWithOutScriptPushdown() throws IOExceptio // Only for Calcite @Test public void testExplainOnEarliestLatest() throws IOException { - String expected = loadExpectedPlan("explain_earliest_latest.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_earliest_latest.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s | stats earliest(message) as earliest_message, latest(message) as" + " latest_message by server", @@ -657,10 +641,10 @@ public void testExplainOnEarliestLatest() throws IOException { // Only for Calcite @Test public void testExplainOnEarliestLatestWithCustomTimeField() throws IOException { - String expected = loadExpectedPlan("explain_earliest_latest_custom_time.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_earliest_latest_custom_time.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s | stats earliest(message, created_at) as earliest_message," + " latest(message, created_at) as latest_message by level", @@ -670,10 +654,10 @@ public void testExplainOnEarliestLatestWithCustomTimeField() throws IOException // Only for Calcite @Test public void testExplainOnFirstLast() throws IOException { - String expected = loadExpectedPlan("explain_first_last.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_first_last.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s | stats first(firstname) as first_name, last(firstname) as" + " last_name by gender", diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index ee979900c48..644ed769ce8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -908,6 +908,11 @@ public enum Index { "time_data", getMappingFile("time_test_data_index_mapping.json"), "src/test/resources/time_test_data.json"), + TIME_BINS_TEST_DATA( + "opensearch-sql_test_index_time_bins_data", + "time_bins_data", + getMappingFile("time_test_data_index_mapping.json"), + "src/test/resources/time_bins_test_data.json"), TIME_TEST_DATA_WITH_NULL( TestsConstants.TEST_INDEX_TIME_DATE_NULL, "time_data_with_null", diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java index ef56ac86fcb..9a7c60d8e39 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java @@ -199,10 +199,10 @@ public void testSortWithTypePushDownExplain() throws IOException { @Test public void testSortWithAggregationExplain() throws IOException { // Sorts whose by fields are aggregators should not be pushed down - String expected = loadExpectedPlan("explain_sort_agg_push.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_sort_agg_push.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| stats avg(age) AS avg_age by state, city " + "| sort avg_age")); @@ -227,10 +227,10 @@ public void testMultiSortPushDownExplain() throws IOException { @Test public void testSortThenAggregatePushDownExplain() throws IOException { - String expected = loadExpectedPlan("explain_sort_then_agg_push.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_sort_then_agg_push.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account" + "| sort balance, age " + "| stats avg(balance) by state")); diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml index 722be1bec9f..0d6642e3a67 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml @@ -1,10 +1,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(count()=[$1], c1=[$1], gender=[$0]) + LogicalProject(c1=[$1], count()=[$1], gender=[$0]) LogicalAggregate(group=[{0}], count()=[COUNT()]) LogicalProject(gender=[$4]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], c1=[$t1], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..1=[{inputs}], c1=[$t1], count()=[$t1], gender=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml index 61724171f7a..fa2b45c8526 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml @@ -1,10 +1,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(c1=[$1], c2=[$1], gender=[$0]) + LogicalProject(c2=[$1], c1=[$1], gender=[$0]) LogicalAggregate(group=[{0}], c1=[COUNT($1)]) LogicalProject(gender=[$4], balance=[$3]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], c1=[$t1], c2=[$t1], gender=[$t0]) + EnumerableCalc(expr#0..1=[{inputs}], c2=[$t1], c1=[$t1], gender=[$t0]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c1=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"c1":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by3.yaml index a4dfdb25064..505f2834488 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by3.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by3.yaml @@ -1,7 +1,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(count(account_number)=[$1], c2=[$2], gender=[$0]) + LogicalProject(c2=[$1], count(account_number)=[$2], gender=[$0]) LogicalAggregate(group=[{0}], count(account_number)=[COUNT($1)], c2=[COUNT($2)]) LogicalProject(gender=[$4], account_number=[$0], account_number_alias=[$0]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by4.yaml index e56eb5ad662..a7297324946 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by4.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by4.yaml @@ -1,7 +1,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(count()=[$1], count(account_number)=[$2], gender=[$0]) + LogicalProject(count(account_number)=[$1], count()=[$2], gender=[$0]) LogicalAggregate(group=[{0}], count()=[COUNT()], count(account_number)=[COUNT($1)]) LogicalProject(gender=[$4], account_number=[$0]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by5.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by5.yaml index dc7bf3629f2..e5240ab1740 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by5.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by5.yaml @@ -1,7 +1,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(count(balance)=[$1], count(account_number)=[$2], gender=[$0]) + LogicalProject(count(account_number)=[$1], count(balance)=[$2], gender=[$0]) LogicalAggregate(group=[{0}], count(balance)=[COUNT($1)], count(account_number)=[COUNT($2)]) LogicalProject(gender=[$4], balance=[$3], account_number=[$0]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml index 5d89382a823..130af48c2e8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(count(b_1)=[$1], c3=[$2], gender=[$0]) + LogicalProject(c3=[$1], count(b_1)=[$2], gender=[$0]) LogicalAggregate(group=[{0}], count(b_1)=[COUNT($1)], c3=[COUNT($2)]) LogicalProject(gender=[$4], b_1=[+($3, 1)], $f3=[POWER($3, 2)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(b_1)=COUNT($1),c3=COUNT($2)), PROJECT->[count(b_1), c3, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(b_1)":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBAXsKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDMsCiAgICAgICJuYW1lIjogIiQzIgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAABh3CAAAACAAAAARdAAIX3JvdXRpbmd+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAZTVFJJTkd0AA5hY2NvdW50X251bWJlcn5xAH4ACnQABExPTkd0AAlmaXJzdG5hbWVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AE3hwfnEAfgAKdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAt0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAeeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAgAAAAAHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgAUcQB+AAx+cQB+ABp0AAdLZXl3b3JkcQB+AB94dAAHYWRkcmVzc3NxAH4AEnEAfgAYcQB+ABtxAH4AH3NxAH4AAAAAAAN3BAAAAAB4dAAGZ2VuZGVyc3EAfgAScQB+ABhxAH4AG3EAfgAfc3EAfgAAAAAAA3cEAAAAAnEAfgAkcQB+ACV4dAAGX2luZGV4cQB+AAx0AARjaXR5c3EAfgAScQB+ABhxAH4AG3EAfgAfc3EAfgAAAAAAA3cEAAAAAnEAfgAkcQB+ACV4dAAJX21heHNjb3JlfnEAfgAKdAAFRkxPQVR0AAZfc2NvcmVxAH4AM3QABV9zb3J0cQB+AA90AAhsYXN0bmFtZXNxAH4AEnEAfgAYcQB+ABtxAH4AH3NxAH4AAAAAAAN3BAAAAAJxAH4AJHEAfgAleHQAB2JhbGFuY2VxAH4AD3QACGVtcGxveWVyc3EAfgAScQB+ABhxAH4AG3EAfgAfc3EAfgAAAAAAA3cEAAAAAnEAfgAkcQB+ACV4dAAFc3RhdGVzcQB+ABJxAH4AGHEAfgAbcQB+AB9zcQB+AAAAAAADdwQAAAACcQB+ACRxAH4AJXh0AANfaWRxAH4ADHQAA2FnZXEAfgAPdAAFZW1haWxzcQB+ABJxAH4AGHEAfgAbcQB+AB9zcQB+AAAAAAADdwQAAAACcQB+ACRxAH4AJXh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}},"c3":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBEXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJQT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiAzLAogICAgICAibmFtZSI6ICIkMyIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAYdwgAAAAgAAAAEXQACF9yb3V0aW5nfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAGU1RSSU5HdAAOYWNjb3VudF9udW1iZXJ+cQB+AAp0AARMT05HdAAJZmlyc3RuYW1lc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+ABN4cH5xAH4ACnQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgALdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AHnhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AIAAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AFHEAfgAMfnEAfgAadAAHS2V5d29yZHEAfgAfeHQAB2FkZHJlc3NzcQB+ABJxAH4AGHEAfgAbcQB+AB9zcQB+AAAAAAADdwQAAAAAeHQABmdlbmRlcnNxAH4AEnEAfgAYcQB+ABtxAH4AH3NxAH4AAAAAAAN3BAAAAAJxAH4AJHEAfgAleHQABl9pbmRleHEAfgAMdAAEY2l0eXNxAH4AEnEAfgAYcQB+ABtxAH4AH3NxAH4AAAAAAAN3BAAAAAJxAH4AJHEAfgAleHQACV9tYXhzY29yZX5xAH4ACnQABUZMT0FUdAAGX3Njb3JlcQB+ADN0AAVfc29ydHEAfgAPdAAIbGFzdG5hbWVzcQB+ABJxAH4AGHEAfgAbcQB+AB9zcQB+AAAAAAADdwQAAAACcQB+ACRxAH4AJXh0AAdiYWxhbmNlcQB+AA90AAhlbXBsb3llcnNxAH4AEnEAfgAYcQB+ABtxAH4AH3NxAH4AAAAAAAN3BAAAAAJxAH4AJHEAfgAleHQABXN0YXRlc3EAfgAScQB+ABhxAH4AG3EAfgAfc3EAfgAAAAAAA3cEAAAAAnEAfgAkcQB+ACV4dAADX2lkcQB+AAx0AANhZ2VxAH4AD3QABWVtYWlsc3EAfgAScQB+ABhxAH4AG3EAfgAfc3EAfgAAAAAAA3cEAAAAAnEAfgAkcQB+ACV4eHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(b_1)=COUNT($1),c3=COUNT($2)), PROJECT->[count(b_1), c3, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(b_1)":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBAXsKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDMsCiAgICAgICJuYW1lIjogIiQzIgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAABh3CAAAACAAAAARdAAIX3JvdXRpbmd+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAZTVFJJTkd0AA5hY2NvdW50X251bWJlcn5xAH4ACnQABExPTkd0AAlmaXJzdG5hbWVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AE3hwfnEAfgAKdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAt0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAeeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAgAAAAAHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgAUcQB+AAx+cQB+ABp0AAdLZXl3b3JkcQB+AB94dAAHYWRkcmVzc3NxAH4AEnEAfgAYcQB+ABtxAH4AH3NxAH4AAAAAAAN3BAAAAAB4dAAGZ2VuZGVyc3EAfgAScQB+ABhxAH4AG3EAfgAfc3EAfgAAAAAAA3cEAAAAAnEAfgAkcQB+ACV4dAAGX2luZGV4cQB+AAx0AARjaXR5c3EAfgAScQB+ABhxAH4AG3EAfgAfc3EAfgAAAAAAA3cEAAAAAnEAfgAkcQB+ACV4dAAJX21heHNjb3JlfnEAfgAKdAAFRkxPQVR0AAZfc2NvcmVxAH4AM3QABV9zb3J0cQB+AA90AAhsYXN0bmFtZXNxAH4AEnEAfgAYcQB+ABtxAH4AH3NxAH4AAAAAAAN3BAAAAAJxAH4AJHEAfgAleHQAB2JhbGFuY2VxAH4AD3QACGVtcGxveWVyc3EAfgAScQB+ABhxAH4AG3EAfgAfc3EAfgAAAAAAA3cEAAAAAnEAfgAkcQB+ACV4dAAFc3RhdGVzcQB+ABJxAH4AGHEAfgAbcQB+AB9zcQB+AAAAAAADdwQAAAACcQB+ACRxAH4AJXh0AANfaWRxAH4ADHQAA2FnZXEAfgAPdAAFZW1haWxzcQB+ABJxAH4AGHEAfgAbcQB+AB9zcQB+AAAAAAADdwQAAAACcQB+ACRxAH4AJXh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}},"c3":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBEXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJQT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiAzLAogICAgICAibmFtZSI6ICIkMyIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAYdwgAAAAgAAAAEXQACF9yb3V0aW5nfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAGU1RSSU5HdAAOYWNjb3VudF9udW1iZXJ+cQB+AAp0AARMT05HdAAJZmlyc3RuYW1lc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+ABN4cH5xAH4ACnQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgALdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AHnhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AIAAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AFHEAfgAMfnEAfgAadAAHS2V5d29yZHEAfgAfeHQAB2FkZHJlc3NzcQB+ABJxAH4AGHEAfgAbcQB+AB9zcQB+AAAAAAADdwQAAAAAeHQABmdlbmRlcnNxAH4AEnEAfgAYcQB+ABtxAH4AH3NxAH4AAAAAAAN3BAAAAAJxAH4AJHEAfgAleHQABl9pbmRleHEAfgAMdAAEY2l0eXNxAH4AEnEAfgAYcQB+ABtxAH4AH3NxAH4AAAAAAAN3BAAAAAJxAH4AJHEAfgAleHQACV9tYXhzY29yZX5xAH4ACnQABUZMT0FUdAAGX3Njb3JlcQB+ADN0AAVfc29ydHEAfgAPdAAIbGFzdG5hbWVzcQB+ABJxAH4AGHEAfgAbcQB+AB9zcQB+AAAAAAADdwQAAAACcQB+ACRxAH4AJXh0AAdiYWxhbmNlcQB+AA90AAhlbXBsb3llcnNxAH4AEnEAfgAYcQB+ABtxAH4AH3NxAH4AAAAAAAN3BAAAAAJxAH4AJHEAfgAleHQABXN0YXRlc3EAfgAScQB+ABhxAH4AG3EAfgAfc3EAfgAAAAAAA3cEAAAAAnEAfgAkcQB+ACV4dAADX2lkcQB+AAx0AANhZ2VxAH4AD3QABWVtYWlsc3EAfgAScQB+ABhxAH4AG3EAfgAfc3EAfgAAAAAAA3cEAAAAAnEAfgAkcQB+ACV4eHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml index f3cd99dacbc..0a8c0f5d7ca 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml @@ -1,10 +1,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(sum(balance)=[$1], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0]) + LogicalProject(sum(balance / 100)=[$1], sum(balance * 100)=[$2], sum(balance - 100)=[$3], sum(balance + 100)=[$4], sum(balance)=[$5], gender=[$0]) LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)=[SUM($2)], sum(balance - 100)=[SUM($3)], sum(balance * 100)=[SUM($4)], sum(balance / 100)=[SUM($5)]) LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance / 100)=[$t1], sum(balance * 100)=[$t6], sum(balance - 100)=[$t7], sum(balance + 100)=[$t8], sum(balance)=[$t3], gender=[$t0]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0Ac97CiAgIm9wIjogewogICAgIm5hbWUiOiAiRElWSURFIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDcsCiAgICAgICJuYW1lIjogIiQ3IgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAxMDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdLAogICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAibnVsbGFibGUiOiB0cnVlCiAgfSwKICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgImR5bmFtaWMiOiBmYWxzZQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAABh3CAAAACAAAAATdAAIX3JvdXRpbmd+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAZTVFJJTkd0AA5hY2NvdW50X251bWJlcn5xAH4ACnQABExPTkd0AAlmaXJzdG5hbWVxAH4ADHQAB2FkZHJlc3NzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AFHhwfnEAfgAKdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAt0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAfeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAhAAAAAHNxAH4AAAAAAAN3BAAAAAB4dAAJYmlydGhkYXRlc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0ZVR5cGWeLVKuEH3KrwIAAUwAB2Zvcm1hdHN0ABBMamF2YS91dGlsL0xpc3Q7eHEAfgAVfnEAfgAKdAAJVElNRVNUQU1QfnEAfgAbdAAERGF0ZXEAfgAgc3EAfgAAAAAAAXcEAAAAAHh0AAZnZW5kZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AFXEAfgAMfnEAfgAbdAAHS2V5d29yZHEAfgAgeHQABl9pbmRleHEAfgAMdAAEY2l0eXEAfgAMdAAJX21heHNjb3JlfnEAfgAKdAAFRkxPQVR0AAZfc2NvcmVxAH4AOHQABV9zb3J0cQB+AA90AAhsYXN0bmFtZXEAfgAMdAAHYmFsYW5jZXEAfgAPdAAIZW1wbG95ZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABXN0YXRlc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnEAfgAxcQB+ADJ4dAADX2lkcQB+AAx0AANhZ2V+cQB+AAp0AAdJTlRFR0VSdAAFZW1haWxzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABG1hbGV+cQB+AAp0AAdCT09MRUFOeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_bins.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_bins.json deleted file mode 100644 index ff327963630..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_bins.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[WIDTH_BUCKET($8, 3, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[3], expr#14=[-($t11, $t12)], expr#15=[WIDTH_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n"}} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_bins.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_bins.yaml new file mode 100644 index 00000000000..115dbb6620e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_bins.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[WIDTH_BUCKET($8, 3, MIN($8) OVER (), MAX($8) OVER ())]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[3], expr#14=[WIDTH_BUCKET($t8, $t13, $t11, $t12)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t14]) + EnumerableLimit(fetch=[5]) + EnumerableWindow(window#0=[window(aggs [MIN($8), MAX($8)])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push4.yaml index b191f90b170..6253ca05d42 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push4.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push4.yaml @@ -1,9 +1,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(c1=[$0], c2=[$0]) + LogicalProject(c2=[$0], c1=[$0]) LogicalAggregate(group=[{}], c1=[COUNT()]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0=[{inputs}], c1=[$t0], c2=[$t0]) + EnumerableCalc(expr#0=[{inputs}], c2=[$t0], c1=[$t0]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},c1=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push5.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push5.yaml index a89a1923ee2..d89d54c0008 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push5.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push5.yaml @@ -1,11 +1,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(c1=[$0], c2=[$0]) + LogicalProject(c2=[$0], c1=[$0]) LogicalAggregate(group=[{}], c1=[COUNT($0)]) LogicalProject(lastname=[$10]) LogicalFilter(condition=[IS NOT NULL($10)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0=[{inputs}], c1=[$t0], c2=[$t0]) + EnumerableCalc(expr#0=[{inputs}], c2=[$t0], c1=[$t0]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($10), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},c1=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"lastname","boost":1.0}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.yaml new file mode 100644 index 00000000000..e184ae6f7f5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(latest_message=[$1], earliest_message=[$2], server=[$0]) + LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)]) + LogicalProject(server=[$1], message=[$3], @timestamp=[$2]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},earliest_message=ARG_MIN($1, $2),latest_message=ARG_MAX($1, $2)), PROJECT->[earliest_message, latest_message, server], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"server":{"terms":{"field":"server","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"earliest_message":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["message"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc"}}]}},"latest_message":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["message"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.yaml new file mode 100644 index 00000000000..7c360c58412 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(latest_message=[$1], earliest_message=[$2], level=[$0]) + LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)]) + LogicalProject(level=[$4], message=[$3], created_at=[$0]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},earliest_message=ARG_MIN($1, $2),latest_message=ARG_MAX($1, $2)), PROJECT->[earliest_message, latest_message, level], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"level":{"terms":{"field":"level","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"earliest_message":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["message"],"excludes":[]},"sort":[{"created_at":{"order":"asc"}}]}},"latest_message":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["message"],"excludes":[]},"sort":[{"created_at":{"order":"desc"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.yaml index c4f023585c2..bd250afdc6d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.yaml @@ -1,10 +1,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(avg_age=[$2], state=[$0], city=[$1]) + LogicalProject(avg_age=[ROUND($2, 2)], state=[$0], city=[$1]) LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) LogicalProject(state=[$7], city=[$5], age=[$8]) LogicalFilter(condition=[>($8, 30)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[city, state, age], FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state, city], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"_source":{"includes":["city","state","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[2], expr#4=[ROUND($t2, $t3)], avg_age=[$t4], state=[$t0], city=[$t1]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[city, state, age], FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"_source":{"includes":["city","state","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.yaml new file mode 100644 index 00000000000..63ddae61af7 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(last_name=[$1], first_name=[$2], gender=[$0]) + LogicalAggregate(group=[{0}], first_name=[FIRST($1)], last_name=[LAST($1)]) + LogicalProject(gender=[$4], firstname=[$1]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},first_name=FIRST($1),last_name=LAST($1)), PROJECT->[first_name, last_name, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"first_name":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["firstname"],"excludes":[]}}},"last_name":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["firstname"],"excludes":[]},"sort":[{"_doc":{"order":"desc"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_output.yaml index 195971e53bc..444c3966887 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_output.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output.yaml @@ -7,7 +7,7 @@ calcite: LogicalFilter(condition=[IS NOT NULL($2)]) LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]) LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) - LogicalProject(avg_age=[$2], state=[$0], city=[$1]) + LogicalProject(avg_age=[ROUND($2, 2)], state=[$0], city=[$1]) LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) LogicalProject(state=[$7], city=[$5], age=[$8]) LogicalFilter(condition=[>($8, 30)]) @@ -17,5 +17,5 @@ calcite: EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[<=($t1, $t2)], proj#0..1=[{exprs}], $condition=[$t3]) EnumerableWindow(window#0=[window(partition {0} order by [0] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], expr#3=[IS NOT NULL($t0)], age2=[$t2], $condition=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[city, state, age], FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), SORT->[0 ASC FIRST], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"_source":{"includes":["city","state","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[ROUND($t0, $t1)], expr#3=[+($t2, $t1)], expr#4=[IS NOT NULL($t2)], age2=[$t3], $condition=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[city, state, age], FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), SORT->[0 ASC FIRST], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"_source":{"includes":["city","state","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_agg_push.yaml new file mode 100644 index 00000000000..5d12cbc982b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_agg_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC-nulls-first]) + LogicalProject(avg_age=[ROUND($2, 2)], state=[$0], city=[$1]) + LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) + LogicalProject(state=[$7], city=[$5], age=[$8]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[2], expr#4=[ROUND($t2, $t3)], avg_age=[$t4], state=[$t0], city=[$t1]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.yaml new file mode 100644 index 00000000000..e2aaef62289 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(balance)=[ROUND($1, 2)], state=[$0]) + LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) + LogicalProject(state=[$7], balance=[$3]) + LogicalSort(sort0=[$3], sort1=[$8], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[2], expr#3=[ROUND($t1, $t2)], avg(balance)=[$t3], state=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[balance, state, age], AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","_source":{"includes":["balance","state","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time.yaml index b3f3f5aed9b..b017e6f9e07 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time.yaml @@ -3,9 +3,9 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(count()=[$1], @timestamp=[$0]) LogicalAggregate(group=[{0}], count()=[COUNT()]) - LogicalProject(@timestamp=[WIDTH_BUCKET($1, 3, -(MAX($1) OVER (), MIN($1) OVER ()), MAX($1) OVER ())]) + LogicalProject(@timestamp=[WIDTH_BUCKET($1, 3, MIN($1) OVER (), MAX($1) OVER ())]) CalciteLogicalIndexScan(table=[[OpenSearch, events]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..1=[{inputs}], expr#2=[0], expr#3=[>($t1, $t2)], count()=[$t1], @timestamp=[$t0], $condition=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time2.yaml index a0080e88f90..fffd9d7cc98 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time2.yaml @@ -1,11 +1,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(avg(cpu_usage)=[$1], @timestamp=[$0]) + LogicalProject(avg(cpu_usage)=[ROUND($1, 2)], @timestamp=[$0]) LogicalAggregate(group=[{0}], avg(cpu_usage)=[AVG($1)]) - LogicalProject(@timestamp=[WIDTH_BUCKET($1, 3, -(MAX($1) OVER (), MIN($1) OVER ()), MAX($1) OVER ())], cpu_usage=[$7]) + LogicalProject(@timestamp=[WIDTH_BUCKET($1, 3, MIN($1) OVER (), MAX($1) OVER ())], cpu_usage=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, events]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NOT NULL($t1)], avg(cpu_usage)=[$t1], @timestamp=[$t0], $condition=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(cpu_usage)=AVG($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[2], expr#3=[ROUND($t1, $t2)], expr#4=[IS NOT NULL($t1)], avg(cpu_usage)=[$t3], @timestamp=[$t0], $condition=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(cpu_usage)=AVG($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term.yaml index 8d3e77e622e..8b0c6325be0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term.yaml @@ -1,11 +1,16 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(count()=[$2], @timestamp=[$0], region=[$1]) - LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) - LogicalProject(@timestamp=[$15], region=[$7]) - LogicalFilter(condition=[AND(IS NOT NULL($15), IS NOT NULL($7))]) - LogicalProject(environment=[$0], status_code=[$2], service=[$3], host=[$4], memory_usage=[$5], response_time=[$6], cpu_usage=[$7], region=[$8], bytes_sent=[$9], _id=[$10], _index=[$11], _score=[$12], _maxscore=[$13], _sort=[$14], _routing=[$15], @timestamp=[WIDTH_BUCKET($1, 3, -(MAX($1) OVER (), MIN($1) OVER ()), MAX($1) OVER ())]) + LogicalProject(count()=[$2], @timestamp=[$3], region=[$1]) + LogicalAggregate(group=[{0, 1}], count()=[COUNT()], @timestamp=[MIN($2)]) + LogicalProject(@timestamp_bin=[$16], region=[$8], @timestamp=[$1]) + LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($8))]) + LogicalProject(environment=[$0], @timestamp=[$1], status_code=[$2], service=[$3], host=[$4], memory_usage=[$5], response_time=[$6], cpu_usage=[$7], region=[$8], bytes_sent=[$9], _id=[$10], _index=[$11], _score=[$12], _maxscore=[$13], _sort=[$14], _routing=[$15], @timestamp_bin=[WIDTH_BUCKET($1, 3, MIN($1) OVER (), MAX($1) OVER ())]) CalciteLogicalIndexScan(table=[[OpenSearch, events]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), @timestamp, region], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"region":{"terms":{"field":"region","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":{"_key":"asc"}},"aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], count()=[$t2], @timestamp=[$t3], region=[$t0]) + EnumerableAggregate(group=[{1, 2}], count()=[COUNT()], @timestamp=[MIN($0)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[3], expr#5=[WIDTH_BUCKET($t0, $t4, $t2, $t3)], expr#6=[IS NOT NULL($t0)], expr#7=[IS NOT NULL($t1)], expr#8=[AND($t6, $t7)], proj#0..1=[{exprs}], @timestamp_bin=[$t5], $condition=[$t8]) + EnumerableWindow(window#0=[window(aggs [MIN($0), MAX($0)])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, region]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["@timestamp","region"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term2.yaml index ffc24ee8939..1bf7f0fa4b4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term2.yaml @@ -1,11 +1,16 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(avg(cpu_usage)=[$2], @timestamp=[$0], region=[$1]) - LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[AVG($2)]) - LogicalProject(@timestamp=[$15], region=[$7], cpu_usage=[$6]) - LogicalFilter(condition=[AND(IS NOT NULL($15), IS NOT NULL($7))]) - LogicalProject(environment=[$0], status_code=[$2], service=[$3], host=[$4], memory_usage=[$5], response_time=[$6], cpu_usage=[$7], region=[$8], bytes_sent=[$9], _id=[$10], _index=[$11], _score=[$12], _maxscore=[$13], _sort=[$14], _routing=[$15], @timestamp=[WIDTH_BUCKET($1, 3, -(MAX($1) OVER (), MIN($1) OVER ()), MAX($1) OVER ())]) + LogicalProject(avg(cpu_usage)=[ROUND($2, 2)], @timestamp=[$3], region=[$1]) + LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[AVG($2)], @timestamp=[MIN($3)]) + LogicalProject(@timestamp_bin=[$16], region=[$8], cpu_usage=[$7], @timestamp=[$1]) + LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($8))]) + LogicalProject(environment=[$0], @timestamp=[$1], status_code=[$2], service=[$3], host=[$4], memory_usage=[$5], response_time=[$6], cpu_usage=[$7], region=[$8], bytes_sent=[$9], _id=[$10], _index=[$11], _score=[$12], _maxscore=[$13], _sort=[$14], _routing=[$15], @timestamp_bin=[WIDTH_BUCKET($1, 3, MIN($1) OVER (), MAX($1) OVER ())]) CalciteLogicalIndexScan(table=[[OpenSearch, events]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1, 2},avg(cpu_usage)=AVG($0)), PROJECT->[avg(cpu_usage), @timestamp, region], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"region":{"terms":{"field":"region","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":{"_key":"asc"}},"aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:DOUBLE], expr#8=[CASE($t6, $t7, $t2)], expr#9=[/($t8, $t3)], expr#10=[2], expr#11=[ROUND($t9, $t10)], avg(cpu_usage)=[$t11], @timestamp=[$t4], region=[$t1]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)], @timestamp=[MIN($3)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[3], expr#6=[WIDTH_BUCKET($t0, $t5, $t3, $t4)], expr#7=[IS NOT NULL($t0)], expr#8=[IS NOT NULL($t2)], expr#9=[AND($t7, $t8)], @timestamp_bin=[$t6], region=[$t2], cpu_usage=[$t1], @timestamp=[$t0], $condition=[$t9]) + EnumerableWindow(window#0=[window(aggs [MIN($0), MAX($0)])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, cpu_usage, region]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["@timestamp","cpu_usage","region"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml index bf861c337b9..0cada89636c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml @@ -1,13 +1,13 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(sum(balance)=[$1], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0]) + LogicalProject(sum(balance / 100)=[$1], sum(balance * 100)=[$2], sum(balance - 100)=[$3], sum(balance + 100)=[$4], sum(balance)=[$5], gender=[$0]) LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)=[SUM($2)], sum(balance - 100)=[SUM($3)], sum(balance * 100)=[SUM($4)], sum(balance / 100)=[SUM($5)]) LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance / 100)=[$t1], sum(balance * 100)=[$t6], sum(balance - 100)=[$t7], sum(balance + 100)=[$t8], sum(balance)=[$t3], gender=[$t0]) EnumerableAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)_COUNT=[COUNT($1)], sum(balance / 100)=[SUM($2)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[100], expr#20=[DIVIDE($t7, $t19)], gender=[$t4], balance=[$t7], $f5=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_bins.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_bins.json deleted file mode 100644 index bbdde96acf1..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_bins.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[WIDTH_BUCKET($8, 3, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[3], expr#14=[-($t11, $t12)], expr#15=[WIDTH_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n"}} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_bins.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_bins.yaml new file mode 100644 index 00000000000..7e8f1e2a8cf --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_bins.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[WIDTH_BUCKET($8, 3, MIN($8) OVER (), MAX($8) OVER ())]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[3], expr#14=[WIDTH_BUCKET($t8, $t13, $t11, $t12)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t14]) + EnumerableLimit(fetch=[5]) + EnumerableWindow(window#0=[window(aggs [MIN($8), MAX($8)])]) + EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest.yaml new file mode 100644 index 00000000000..0206d0f9e49 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(latest_message=[$1], earliest_message=[$2], server=[$0]) + LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)]) + LogicalProject(server=[$1], message=[$3], @timestamp=[$2]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], latest_message=[$t1], earliest_message=[$t2], server=[$t0]) + EnumerableAggregate(group=[{1}], earliest_message=[ARG_MIN($3, $2)], latest_message=[ARG_MAX($3, $2)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest_custom_time.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest_custom_time.yaml new file mode 100644 index 00000000000..517874820b0 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest_custom_time.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(latest_message=[$1], earliest_message=[$2], level=[$0]) + LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)]) + LogicalProject(level=[$4], message=[$3], created_at=[$0]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], latest_message=[$t1], earliest_message=[$t2], level=[$t0]) + EnumerableAggregate(group=[{4}], earliest_message=[ARG_MIN($3, $0)], latest_message=[ARG_MAX($3, $0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml index ac3728eacb9..788d6964669 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml @@ -1,14 +1,14 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(avg_age=[$2], state=[$0], city=[$1]) + LogicalProject(avg_age=[ROUND($2, 2)], state=[$0], city=[$1]) LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) LogicalProject(state=[$7], city=[$5], age=[$8]) LogicalFilter(condition=[>($8, 30)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_age=[$t9], state=[$t1], city=[$t0]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[2], expr#11=[ROUND($t9, $t10)], avg_age=[$t11], state=[$t1], city=[$t0]) EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_first_last.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_first_last.yaml new file mode 100644 index 00000000000..220a5f47d16 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_first_last.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(last_name=[$1], first_name=[$2], gender=[$0]) + LogicalAggregate(group=[{0}], first_name=[FIRST($1)], last_name=[LAST($1)]) + LogicalProject(gender=[$4], firstname=[$1]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], last_name=[$t1], first_name=[$t2], gender=[$t0]) + EnumerableAggregate(group=[{4}], first_name=[FIRST($1)], last_name=[LAST($1)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml index 8c0283f0e9c..110b5611282 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml @@ -7,7 +7,7 @@ calcite: LogicalFilter(condition=[IS NOT NULL($2)]) LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]) LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) - LogicalProject(avg_age=[$2], state=[$0], city=[$1]) + LogicalProject(avg_age=[ROUND($2, 2)], state=[$0], city=[$1]) LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) LogicalProject(state=[$7], city=[$5], age=[$8]) LogicalFilter(condition=[>($8, 30)]) @@ -17,7 +17,7 @@ calcite: EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) EnumerableWindow(window#0=[window(partition {1} order by [1] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[2], expr#11=[+($t9, $t10)], expr#12=[IS NOT NULL($t8)], state=[$t1], age2=[$t11], $condition=[$t12]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[2], expr#11=[ROUND($t9, $t10)], expr#12=[+($t11, $t10)], expr#13=[IS NOT NULL($t11)], state=[$t1], age2=[$t12], $condition=[$t13]) EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first]) EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_agg_push.yaml new file mode 100644 index 00000000000..357a36f83b7 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_agg_push.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC-nulls-first]) + LogicalProject(avg_age=[ROUND($2, 2)], state=[$0], city=[$1]) + LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) + LogicalProject(state=[$7], city=[$5], age=[$8]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[2], expr#11=[ROUND($t9, $t10)], avg_age=[$t11], state=[$t1], city=[$t0]) + EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.yaml new file mode 100644 index 00000000000..ca194329648 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(balance)=[ROUND($1, 2)], state=[$0]) + LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) + LogicalProject(state=[$7], balance=[$3]) + LogicalSort(sort0=[$3], sort1=[$8], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], expr#9=[2], expr#10=[ROUND($t8, $t9)], avg(balance)=[$t10], state=[$t0]) + EnumerableAggregate(group=[{7}], agg#0=[$SUM0($3)], agg#1=[COUNT($3)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/time_bins_test_data.json b/integ-test/src/test/resources/time_bins_test_data.json new file mode 100644 index 00000000000..fe0bd75e0a6 --- /dev/null +++ b/integ-test/src/test/resources/time_bins_test_data.json @@ -0,0 +1,80 @@ +{"index":{"_id":"1"}} +{"@timestamp":"2025-07-28T10:00:00","value":8945,"category":"A"} +{"index":{"_id":"2"}} +{"@timestamp":"2025-07-28T10:00:17","value":9012,"category":"B"} +{"index":{"_id":"3"}} +{"@timestamp":"2025-07-28T10:00:23","value":6712,"category":"C"} +{"index":{"_id":"4"}} +{"@timestamp":"2025-07-28T10:00:41","value":8917,"category":"D"} +{"index":{"_id":"5"}} +{"@timestamp":"2025-07-28T10:00:58","value":7162,"category":"A"} +{"index":{"_id":"6"}} +{"@timestamp":"2025-07-28T10:01:12","value":8429,"category":"B"} +{"index":{"_id":"7"}} +{"@timestamp":"2025-07-28T10:01:29","value":6985,"category":"C"} +{"index":{"_id":"8"}} +{"@timestamp":"2025-07-28T10:01:47","value":6583,"category":"D"} +{"index":{"_id":"9"}} +{"@timestamp":"2025-07-28T10:02:03","value":7823,"category":"A"} +{"index":{"_id":"10"}} +{"@timestamp":"2025-07-28T10:02:21","value":9156,"category":"B"} +{"index":{"_id":"11"}} +{"@timestamp":"2025-07-28T10:02:38","value":6234,"category":"C"} +{"index":{"_id":"12"}} +{"@timestamp":"2025-07-28T10:02:55","value":8765,"category":"D"} +{"index":{"_id":"13"}} +{"@timestamp":"2025-07-28T10:03:09","value":7491,"category":"A"} +{"index":{"_id":"14"}} +{"@timestamp":"2025-07-28T10:03:26","value":8302,"category":"B"} +{"index":{"_id":"15"}} +{"@timestamp":"2025-07-28T10:03:44","value":6874,"category":"C"} +{"index":{"_id":"16"}} +{"@timestamp":"2025-07-28T10:04:01","value":9421,"category":"D"} +{"index":{"_id":"17"}} +{"@timestamp":"2025-07-28T10:04:18","value":7658,"category":"A"} +{"index":{"_id":"18"}} +{"@timestamp":"2025-07-28T10:04:35","value":8193,"category":"B"} +{"index":{"_id":"19"}} +{"@timestamp":"2025-07-28T10:04:52","value":6542,"category":"C"} +{"index":{"_id":"20"}} +{"@timestamp":"2025-07-28T10:05:07","value":8936,"category":"D"} +{"index":{"_id":"21"}} +{"@timestamp":"2025-07-28T10:15:30","value":7200,"category":"A"} +{"index":{"_id":"22"}} +{"@timestamp":"2025-07-28T10:32:45","value":8100,"category":"B"} +{"index":{"_id":"23"}} +{"@timestamp":"2025-07-28T10:48:20","value":6900,"category":"C"} +{"index":{"_id":"24"}} +{"@timestamp":"2025-07-28T11:05:15","value":7500,"category":"D"} +{"index":{"_id":"25"}} +{"@timestamp":"2025-07-28T11:22:40","value":8300,"category":"A"} +{"index":{"_id":"26"}} +{"@timestamp":"2025-07-28T11:38:55","value":6700,"category":"B"} +{"index":{"_id":"27"}} +{"@timestamp":"2025-07-28T11:54:10","value":7800,"category":"C"} +{"index":{"_id":"28"}} +{"@timestamp":"2025-07-28T12:08:25","value":9200,"category":"D"} +{"index":{"_id":"29"}} +{"@timestamp":"2025-07-28T12:25:50","value":6500,"category":"A"} +{"index":{"_id":"30"}} +{"@timestamp":"2025-07-28T12:41:35","value":8700,"category":"B"} +{"index":{"_id":"31"}} +{"@timestamp":"2025-07-28T12:56:20","value":7100,"category":"C"} +{"index":{"_id":"32"}} +{"@timestamp":"2025-07-28T13:12:45","value":8900,"category":"D"} +{"index":{"_id":"33"}} +{"@timestamp":"2025-07-28T13:28:30","value":6300,"category":"A"} +{"index":{"_id":"34"}} +{"@timestamp":"2025-07-28T13:45:15","value":7900,"category":"B"} +{"index":{"_id":"35"}} +{"@timestamp":"2025-07-28T14:02:50","value":8500,"category":"C"} +{"index":{"_id":"36"}} +{"@timestamp":"2025-07-28T14:18:25","value":6800,"category":"D"} +{"index":{"_id":"37"}} +{"@timestamp":"2025-07-28T14:35:40","value":9100,"category":"A"} +{"index":{"_id":"38"}} +{"@timestamp":"2025-07-28T14:52:15","value":7400,"category":"B"} +{"index":{"_id":"39"}} +{"@timestamp":"2025-07-28T15:08:30","value":8200,"category":"C"} +{"index":{"_id":"40"}} +{"@timestamp":"2025-07-28T15:24:55","value":6600,"category":"D"} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLBinTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLBinTest.java index c940a750c28..d7bd829eef7 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLBinTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLBinTest.java @@ -40,11 +40,12 @@ public void testBinWithBins() { // Note: WIDTH_BUCKET uses window functions without ROWS UNBOUNDED PRECEDING in the actual // output + // Updated signature: WIDTH_BUCKET(field, numBins, minValue, maxValue) verifyLogical( root, "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], " + "COMM=[$6], DEPTNO=[$7], SAL=[WIDTH_BUCKET($5, 10, " - + "-(MAX($5) OVER (), MIN($5) OVER ()), " + + "MIN($5) OVER (), " + "MAX($5) OVER ())])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"); }