Skip to content

Commit 303eeac

Browse files
committed
fix tests
Signed-off-by: Kai Huang <[email protected]>
1 parent 6fdcd9f commit 303eeac

File tree

9 files changed

+72
-47
lines changed

9 files changed

+72
-47
lines changed

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java

Lines changed: 60 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -988,8 +988,6 @@ public void testStatsWithBinsOnTimeAndTermField_Avg() throws IOException {
988988

989989
@Test
990990
public void testBinTimestampBins20WithStats() throws IOException {
991-
// TODO: Remove this after addressing https://github.com/opensearch-project/sql/issues/4317
992-
enabledOnlyWhenPushdownIsEnabled();
993991

994992
// Test bins=20 with aggregation to verify OpenSearch auto_date_histogram behavior
995993
// This corresponds to SPL test case 1: bins=20, range=307s
@@ -1173,16 +1171,23 @@ public void testBinTimestampBins20WithoutAggregation() throws IOException {
11731171
executeQuery(
11741172
"source=opensearch-sql_test_index_time_bins_data | where @timestamp < '2025-07-28"
11751173
+ " 10:06:00' | bin @timestamp bins=20 | fields @timestamp, value | sort @timestamp"
1176-
+ " | head 3");
1174+
+ " | head 10");
11771175

11781176
verifySchema(result, schema("@timestamp", null, "timestamp"), schema("value", null, "int"));
11791177

1180-
// All records from 10:00:00-10:00:29 should be binned to 2025-07-28 10:00:00
1178+
// Verify 10 rows with 30-second interval bins
11811179
verifyDataRows(
11821180
result,
11831181
rows("2025-07-28 10:00:00", 8945),
11841182
rows("2025-07-28 10:00:00", 9012),
1185-
rows("2025-07-28 10:00:00", 6712));
1183+
rows("2025-07-28 10:00:00", 6712),
1184+
rows("2025-07-28 10:00:30", 8917),
1185+
rows("2025-07-28 10:00:30", 7162),
1186+
rows("2025-07-28 10:01:00", 8429),
1187+
rows("2025-07-28 10:01:00", 6985),
1188+
rows("2025-07-28 10:01:30", 6583),
1189+
rows("2025-07-28 10:02:00", 7823),
1190+
rows("2025-07-28 10:02:00", 9156));
11861191
}
11871192

11881193
@Test
@@ -1192,16 +1197,23 @@ public void testBinTimestampBins10WithoutAggregation() throws IOException {
11921197
executeQuery(
11931198
"source=opensearch-sql_test_index_time_bins_data | where @timestamp < '2025-07-28"
11941199
+ " 10:06:00' | bin @timestamp bins=10 | fields @timestamp, value | sort @timestamp"
1195-
+ " | head 3");
1200+
+ " | head 10");
11961201

11971202
verifySchema(result, schema("@timestamp", null, "timestamp"), schema("value", null, "int"));
11981203

1199-
// All records from 10:00:00-10:00:59 should be binned to 2025-07-28 10:00:00
1204+
// Verify 10 rows with 1-minute interval bins
12001205
verifyDataRows(
12011206
result,
12021207
rows("2025-07-28 10:00:00", 8945),
12031208
rows("2025-07-28 10:00:00", 9012),
1204-
rows("2025-07-28 10:00:00", 6712));
1209+
rows("2025-07-28 10:00:00", 6712),
1210+
rows("2025-07-28 10:00:00", 8917),
1211+
rows("2025-07-28 10:00:00", 7162),
1212+
rows("2025-07-28 10:01:00", 8429),
1213+
rows("2025-07-28 10:01:00", 6985),
1214+
rows("2025-07-28 10:01:00", 6583),
1215+
rows("2025-07-28 10:02:00", 7823),
1216+
rows("2025-07-28 10:02:00", 9156));
12051217
}
12061218

12071219
@Test
@@ -1211,16 +1223,23 @@ public void testBinTimestampBins5WithoutAggregation() throws IOException {
12111223
executeQuery(
12121224
"source=opensearch-sql_test_index_time_bins_data | where @timestamp < '2025-07-28"
12131225
+ " 10:06:00' | bin @timestamp bins=5 | fields @timestamp, value | sort @timestamp"
1214-
+ " | head 3");
1226+
+ " | head 10");
12151227

12161228
verifySchema(result, schema("@timestamp", null, "timestamp"), schema("value", null, "int"));
12171229

1218-
// All records from 10:00:00-10:04:59 should be binned to 2025-07-28 10:00:00
1230+
// Verify 10 rows with 5-minute interval bins (all in same bin)
12191231
verifyDataRows(
12201232
result,
12211233
rows("2025-07-28 10:00:00", 8945),
12221234
rows("2025-07-28 10:00:00", 9012),
1223-
rows("2025-07-28 10:00:00", 6712));
1235+
rows("2025-07-28 10:00:00", 6712),
1236+
rows("2025-07-28 10:00:00", 8917),
1237+
rows("2025-07-28 10:00:00", 7162),
1238+
rows("2025-07-28 10:00:00", 8429),
1239+
rows("2025-07-28 10:00:00", 6985),
1240+
rows("2025-07-28 10:00:00", 6583),
1241+
rows("2025-07-28 10:00:00", 7823),
1242+
rows("2025-07-28 10:00:00", 9156));
12241243
}
12251244

12261245
@Test
@@ -1229,16 +1248,23 @@ public void testBinTimestampBins10HoursWithoutAggregation() throws IOException {
12291248
JSONObject result =
12301249
executeQuery(
12311250
"source=opensearch-sql_test_index_time_bins_data | bin @timestamp bins=10 | fields"
1232-
+ " @timestamp, value | sort @timestamp | head 3");
1251+
+ " @timestamp, value | sort @timestamp | head 10");
12331252

12341253
verifySchema(result, schema("@timestamp", null, "timestamp"), schema("value", null, "int"));
12351254

1236-
// Records from 10:00:00-10:59:59 should be binned to 2025-07-28 10:00:00
1255+
// Verify 10 rows with 1-hour interval bins (all in same bin)
12371256
verifyDataRows(
12381257
result,
12391258
rows("2025-07-28 10:00:00", 8945),
12401259
rows("2025-07-28 10:00:00", 9012),
1241-
rows("2025-07-28 10:00:00", 6712));
1260+
rows("2025-07-28 10:00:00", 6712),
1261+
rows("2025-07-28 10:00:00", 8917),
1262+
rows("2025-07-28 10:00:00", 7162),
1263+
rows("2025-07-28 10:00:00", 8429),
1264+
rows("2025-07-28 10:00:00", 6985),
1265+
rows("2025-07-28 10:00:00", 6583),
1266+
rows("2025-07-28 10:00:00", 7823),
1267+
rows("2025-07-28 10:00:00", 9156));
12421268
}
12431269

12441270
@Test
@@ -1247,16 +1273,23 @@ public void testBinTimestampBins5HoursWithoutAggregation() throws IOException {
12471273
JSONObject result =
12481274
executeQuery(
12491275
"source=opensearch-sql_test_index_time_bins_data | bin @timestamp bins=5 | fields"
1250-
+ " @timestamp, value | sort @timestamp | head 3");
1276+
+ " @timestamp, value | sort @timestamp | head 10");
12511277

12521278
verifySchema(result, schema("@timestamp", null, "timestamp"), schema("value", null, "int"));
12531279

1254-
// Records from 10:00:00-12:59:59 should be binned to 2025-07-28 10:00:00
1280+
// Verify 10 rows with 3-hour interval bins (all in same bin)
12551281
verifyDataRows(
12561282
result,
12571283
rows("2025-07-28 10:00:00", 8945),
12581284
rows("2025-07-28 10:00:00", 9012),
1259-
rows("2025-07-28 10:00:00", 6712));
1285+
rows("2025-07-28 10:00:00", 6712),
1286+
rows("2025-07-28 10:00:00", 8917),
1287+
rows("2025-07-28 10:00:00", 7162),
1288+
rows("2025-07-28 10:00:00", 8429),
1289+
rows("2025-07-28 10:00:00", 6985),
1290+
rows("2025-07-28 10:00:00", 6583),
1291+
rows("2025-07-28 10:00:00", 7823),
1292+
rows("2025-07-28 10:00:00", 9156));
12601293
}
12611294

12621295
@Test
@@ -1265,15 +1298,22 @@ public void testBinTimestampBins3HoursWithoutAggregation() throws IOException {
12651298
JSONObject result =
12661299
executeQuery(
12671300
"source=opensearch-sql_test_index_time_bins_data | bin @timestamp bins=3 | fields"
1268-
+ " @timestamp, value | sort @timestamp | head 3");
1301+
+ " @timestamp, value | sort @timestamp | head 10");
12691302

12701303
verifySchema(result, schema("@timestamp", null, "timestamp"), schema("value", null, "int"));
12711304

1272-
// Records from 10:00:00-12:59:59 should be binned to 2025-07-28 10:00:00
1305+
// Verify 10 rows with 3-hour interval bins (all in same bin)
12731306
verifyDataRows(
12741307
result,
12751308
rows("2025-07-28 10:00:00", 8945),
12761309
rows("2025-07-28 10:00:00", 9012),
1277-
rows("2025-07-28 10:00:00", 6712));
1310+
rows("2025-07-28 10:00:00", 6712),
1311+
rows("2025-07-28 10:00:00", 8917),
1312+
rows("2025-07-28 10:00:00", 7162),
1313+
rows("2025-07-28 10:00:00", 8429),
1314+
rows("2025-07-28 10:00:00", 6985),
1315+
rows("2025-07-28 10:00:00", 6583),
1316+
rows("2025-07-28 10:00:00", 7823),
1317+
rows("2025-07-28 10:00:00", 9156));
12781318
}
12791319
}

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java

Lines changed: 1 addition & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -474,7 +474,7 @@ public void supportPushDownScriptOnTextField() throws IOException {
474474

475475
@Test
476476
public void testExplainBinWithBins() throws IOException {
477-
String expected = loadExpectedPlan("explain_bin_bins.json");
477+
String expected = loadExpectedPlan("explain_bin_bins.yaml");
478478
assertJsonEqualsIgnoreId(
479479
expected,
480480
explainQueryToString("source=opensearch-sql_test_index_account | bin age bins=3 | head 5"));
@@ -515,22 +515,6 @@ public void testExplainStatsWithSubAggregation() throws IOException {
515515
+ " @timestamp, region"));
516516
}
517517

518-
@Test
519-
public void bucketNullableNotSupportSubAggregation() throws IOException {
520-
// TODO: Don't throw exception after addressing
521-
// https://github.com/opensearch-project/sql/issues/4317
522-
// When bucketNullable is true, sub aggregation is not supported. Hence we cannot pushdown the
523-
// aggregation in this query. Caused by issue
524-
// https://github.com/opensearch-project/sql/issues/4317,
525-
// bin aggregation on timestamp field won't work if not been push down.
526-
enabledOnlyWhenPushdownIsEnabled();
527-
assertThrows(
528-
Exception.class,
529-
() ->
530-
explainQueryToString(
531-
"source=events | bin @timestamp bins=3 | stats count() by @timestamp, region"));
532-
}
533-
534518
@Test
535519
public void testExplainBinWithSpan() throws IOException {
536520
String expected = loadExpectedPlan("explain_bin_span.yaml");

integ-test/src/test/resources/expectedOutput/calcite/explain_bin_bins.json

Lines changed: 0 additions & 1 deletion
This file was deleted.
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[WIDTH_BUCKET($8, 3, MIN($8) OVER (), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[3], expr#14=[WIDTH_BUCKET($t8, $t13, $t11, $t12)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t14])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MIN($8), MAX($8)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n"}}

integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ calcite:
33
LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])
44
LogicalProject(count()=[$1], @timestamp=[$0])
55
LogicalAggregate(group=[{0}], count()=[COUNT()])
6-
LogicalProject(@timestamp=[WIDTH_BUCKET($1, 3, -(MAX($1) OVER (), MIN($1) OVER ()), MAX($1) OVER ())])
6+
LogicalProject(@timestamp=[WIDTH_BUCKET($1, 3, MIN($1) OVER (), MAX($1) OVER ())])
77
CalciteLogicalIndexScan(table=[[OpenSearch, events]])
88
physical: |
99
EnumerableLimit(fetch=[10000])
1010
EnumerableCalc(expr#0..1=[{inputs}], expr#2=[0], expr#3=[>($t1, $t2)], count()=[$t1], @timestamp=[$t0], $condition=[$t3])
11-
CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])
11+
CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])

integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time2.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ calcite:
33
LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])
44
LogicalProject(avg(cpu_usage)=[$1], @timestamp=[$0])
55
LogicalAggregate(group=[{0}], avg(cpu_usage)=[AVG($1)])
6-
LogicalProject(@timestamp=[WIDTH_BUCKET($1, 3, -(MAX($1) OVER (), MIN($1) OVER ()), MAX($1) OVER ())], cpu_usage=[$7])
6+
LogicalProject(@timestamp=[WIDTH_BUCKET($1, 3, MIN($1) OVER (), MAX($1) OVER ())], cpu_usage=[$7])
77
CalciteLogicalIndexScan(table=[[OpenSearch, events]])
88
physical: |
99
EnumerableLimit(fetch=[10000])
1010
EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NOT NULL($t1)], avg(cpu_usage)=[$t1], @timestamp=[$t0], $condition=[$t2])
11-
CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(cpu_usage)=AVG($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])
11+
CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(cpu_usage)=AVG($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])

integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ calcite:
55
LogicalAggregate(group=[{0, 1}], count()=[COUNT()])
66
LogicalProject(@timestamp=[$15], region=[$7])
77
LogicalFilter(condition=[AND(IS NOT NULL($15), IS NOT NULL($7))])
8-
LogicalProject(environment=[$0], status_code=[$2], service=[$3], host=[$4], memory_usage=[$5], response_time=[$6], cpu_usage=[$7], region=[$8], bytes_sent=[$9], _id=[$10], _index=[$11], _score=[$12], _maxscore=[$13], _sort=[$14], _routing=[$15], @timestamp=[WIDTH_BUCKET($1, 3, -(MAX($1) OVER (), MIN($1) OVER ()), MAX($1) OVER ())])
8+
LogicalProject(environment=[$0], status_code=[$2], service=[$3], host=[$4], memory_usage=[$5], response_time=[$6], cpu_usage=[$7], region=[$8], bytes_sent=[$9], _id=[$10], _index=[$11], _score=[$12], _maxscore=[$13], _sort=[$14], _routing=[$15], @timestamp=[WIDTH_BUCKET($1, 3, MIN($1) OVER (), MAX($1) OVER ())])
99
CalciteLogicalIndexScan(table=[[OpenSearch, events]])
1010
physical: |
11-
CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), @timestamp, region], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"region":{"terms":{"field":"region","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":{"_key":"asc"}},"aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])
11+
CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), @timestamp, region], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"region":{"terms":{"field":"region","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":{"_key":"asc"}},"aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])

integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term2.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ calcite:
55
LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[AVG($2)])
66
LogicalProject(@timestamp=[$15], region=[$7], cpu_usage=[$6])
77
LogicalFilter(condition=[AND(IS NOT NULL($15), IS NOT NULL($7))])
8-
LogicalProject(environment=[$0], status_code=[$2], service=[$3], host=[$4], memory_usage=[$5], response_time=[$6], cpu_usage=[$7], region=[$8], bytes_sent=[$9], _id=[$10], _index=[$11], _score=[$12], _maxscore=[$13], _sort=[$14], _routing=[$15], @timestamp=[WIDTH_BUCKET($1, 3, -(MAX($1) OVER (), MIN($1) OVER ()), MAX($1) OVER ())])
8+
LogicalProject(environment=[$0], status_code=[$2], service=[$3], host=[$4], memory_usage=[$5], response_time=[$6], cpu_usage=[$7], region=[$8], bytes_sent=[$9], _id=[$10], _index=[$11], _score=[$12], _maxscore=[$13], _sort=[$14], _routing=[$15], @timestamp=[WIDTH_BUCKET($1, 3, MIN($1) OVER (), MAX($1) OVER ())])
99
CalciteLogicalIndexScan(table=[[OpenSearch, events]])
1010
physical: |
11-
CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1, 2},avg(cpu_usage)=AVG($0)), PROJECT->[avg(cpu_usage), @timestamp, region], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"region":{"terms":{"field":"region","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":{"_key":"asc"}},"aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])
11+
CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1, 2},avg(cpu_usage)=AVG($0)), PROJECT->[avg(cpu_usage), @timestamp, region], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"region":{"terms":{"field":"region","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":{"_key":"asc"}},"aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])

0 commit comments

Comments
 (0)