Skip to content

Commit 8c3ec27

Browse files
patchwork01patchwork01
authored
Merge branch 'develop' into 4472-reduce-duplication-between-starting-bulk-export-and-compaction-tasks
2 parents a1726f0 + c632e00 commit 8c3ec27

File tree

107 files changed

+5080
-1761
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

107 files changed

+5080
-1761
lines changed

‎example/full/table.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ sleeper.table.iterator.config=b,3600000
3030
# sleeper.table.filters=
3131

3232
# Property to configure aggregation of columns from the table. Format accepted: op(column),op(column)
33-
sleeper.table.aggregations=
33+
# sleeper.table.aggregations=
3434

3535

3636
## The following table properties relate to partition splitting.

‎java/athena/src/main/java/sleeper/athena/metadata/IteratorApplyingMetadataHandler.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import sleeper.core.schema.Field;
3939
import sleeper.core.schema.type.PrimitiveType;
4040

41+
import java.util.Collection;
4142
import java.util.HashSet;
4243
import java.util.List;
4344
import java.util.Set;
@@ -80,7 +81,7 @@ protected void addExtraSchemaEnhancements(SchemaBuilder partitionSchemaBuilder,
8081

8182
@Override
8283
protected void writeExtraPartitionDataToBlock(Partition partition, Block block, int rowNum) {
83-
List<Range> ranges = partition.getRegion().getRanges();
84+
Collection<Range> ranges = partition.getRegion().getRangesUnordered();
8485
for (Range range : ranges) {
8586
block.setValue(MIN_ROW_KEY_PREFIX + "-" + range.getFieldName(), rowNum, range.getMin());
8687
block.setValue(MAX_ROW_KEY_PREFIX + "-" + range.getFieldName(), rowNum, range.getMax());

‎java/clients/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,11 @@
254254
<type>test-jar</type>
255255
<scope>test</scope>
256256
</dependency>
257+
<dependency>
258+
<groupId>com.approvaltests</groupId>
259+
<artifactId>approvaltests</artifactId>
260+
<scope>test</scope>
261+
</dependency>
257262
</dependencies>
258263

259264
<build>

‎java/clients/src/test/java/sleeper/clients/util/ShowPageIndexes-v1-multiplePages.approved.txt‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
Row group 0
2+
Column index for column test-key
3+
Boundary order: ASCENDING
4+
null count min max rep level histogram def level histogram
5+
page-0 0 row-0001 row-0100 <none> <none>
6+
page-1 0 row-0101 row-0200 <none> <none>
7+
page-2 0 row-0201 row-0300 <none> <none>
8+
page-3 0 row-0301 row-0400 <none> <none>
9+
page-4 0 row-0401 row-0500 <none> <none>
10+
page-5 0 row-0501 row-0600 <none> <none>
11+
page-6 0 row-0601 row-0700 <none> <none>
12+
page-7 0 row-0701 row-0800 <none> <none>
13+
page-8 0 row-0801 row-0900 <none> <none>
14+
page-9 0 row-0901 row-1000 <none> <none>
15+
16+
Page index for column test-key
17+
offset compressed size first row index unencoded bytes
18+
page-0 4 144 0 800
19+
page-1 148 144 100 800
20+
page-2 292 144 200 800
21+
page-3 436 144 300 800
22+
page-4 580 144 400 800
23+
page-5 724 144 500 800
24+
page-6 868 144 600 800
25+
page-7 1012 144 700 800
26+
page-8 1156 144 800 800
27+
page-9 1300 145 900 800
28+

‎java/clients/src/test/java/sleeper/clients/util/ShowPageIndexes-v1-multipleRowGroups.approved.txt‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
Row group 0
2+
Column index for column test-key
3+
Boundary order: ASCENDING
4+
null count min max rep level histogram def level histogram
5+
page-0 0 row-0001 row-0100 <none> <none>
6+
7+
Page index for column test-key
8+
offset compressed size first row index unencoded bytes
9+
page-0 4 144 0 800
10+
11+
Row group 1
12+
Column index for column test-key
13+
Boundary order: ASCENDING
14+
null count min max rep level histogram def level histogram
15+
page-0 0 row-0101 row-0200 <none> <none>
16+
17+
Page index for column test-key
18+
offset compressed size first row index unencoded bytes
19+
page-0 148 144 0 800
20+
21+
Row group 2
22+
Column index for column test-key
23+
Boundary order: ASCENDING
24+
null count min max rep level histogram def level histogram
25+
page-0 0 row-0201 row-0300 <none> <none>
26+
27+
Page index for column test-key
28+
offset compressed size first row index unencoded bytes
29+
page-0 292 144 0 800
30+
31+
Row group 3
32+
Column index for column test-key
33+
Boundary order: ASCENDING
34+
null count min max rep level histogram def level histogram
35+
page-0 0 row-0301 row-0400 <none> <none>
36+
37+
Page index for column test-key
38+
offset compressed size first row index unencoded bytes
39+
page-0 436 144 0 800
40+
41+
Row group 4
42+
Column index for column test-key
43+
Boundary order: ASCENDING
44+
null count min max rep level histogram def level histogram
45+
page-0 0 row-0401 row-0500 <none> <none>
46+
47+
Page index for column test-key
48+
offset compressed size first row index unencoded bytes
49+
page-0 580 144 0 800
50+
51+
Row group 5
52+
Column index for column test-key
53+
Boundary order: ASCENDING
54+
null count min max rep level histogram def level histogram
55+
page-0 0 row-0501 row-0600 <none> <none>
56+
57+
Page index for column test-key
58+
offset compressed size first row index unencoded bytes
59+
page-0 724 144 0 800
60+
61+
Row group 6
62+
Column index for column test-key
63+
Boundary order: ASCENDING
64+
null count min max rep level histogram def level histogram
65+
page-0 0 row-0601 row-0700 <none> <none>
66+
67+
Page index for column test-key
68+
offset compressed size first row index unencoded bytes
69+
page-0 868 144 0 800
70+
71+
Row group 7
72+
Column index for column test-key
73+
Boundary order: ASCENDING
74+
null count min max rep level histogram def level histogram
75+
page-0 0 row-0701 row-0800 <none> <none>
76+
77+
Page index for column test-key
78+
offset compressed size first row index unencoded bytes
79+
page-0 1012 144 0 800
80+
81+
Row group 8
82+
Column index for column test-key
83+
Boundary order: ASCENDING
84+
null count min max rep level histogram def level histogram
85+
page-0 0 row-0801 row-0900 <none> <none>
86+
87+
Page index for column test-key
88+
offset compressed size first row index unencoded bytes
89+
page-0 1156 144 0 800
90+
91+
Row group 9
92+
Column index for column test-key
93+
Boundary order: ASCENDING
94+
null count min max rep level histogram def level histogram
95+
page-0 0 row-0901 row-1000 <none> <none>
96+
97+
Page index for column test-key
98+
offset compressed size first row index unencoded bytes
99+
page-0 1300 145 0 800
100+

‎java/clients/src/test/java/sleeper/clients/util/ShowPageIndexes-v1-multipleRowKeyFields.approved.txt‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
Row group 0
2+
Column index for column test-key1
3+
Boundary order: ASCENDING
4+
null count min max rep level histogram def level histogram
5+
page-0 0 row1-001 row1-100 <none> <none>
6+
7+
Page index for column test-key1
8+
offset compressed size first row index unencoded bytes
9+
page-0 4 144 0 800
10+
11+
Column index for column test-key2
12+
Boundary order: ASCENDING
13+
null count min max rep level histogram def level histogram
14+
page-0 0 row2-001 row2-100 <none> <none>
15+
16+
Page index for column test-key2
17+
offset compressed size first row index unencoded bytes
18+
page-0 148 144 0 800
19+

‎java/clients/src/test/java/sleeper/clients/util/ShowPageIndexes-v1-oneRowKeyField.approved.txt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
Row group 0
2+
Column index for column test-key
3+
Boundary order: ASCENDING
4+
null count min max rep level histogram def level histogram
5+
page-0 0 row-001 row-100 <none> <none>
6+
7+
Page index for column test-key
8+
offset compressed size first row index unencoded bytes
9+
page-0 4 142 0 700
10+

‎java/clients/src/test/java/sleeper/clients/util/ShowPageIndexes-v1-rowKeySortKeyAndValueFields.approved.txt‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
Row group 0
2+
Column index for column test-key
3+
Boundary order: ASCENDING
4+
null count min max rep level histogram def level histogram
5+
page-0 0 row-001 row-100 <none> <none>
6+
7+
Page index for column test-key
8+
offset compressed size first row index unencoded bytes
9+
page-0 4 142 0 700
10+
11+
Column index for column test-sort
12+
Boundary order: ASCENDING
13+
null count min max rep level histogram def level histogram
14+
page-0 0 1 100 <none> <none>
15+
16+
Page index for column test-sort
17+
offset compressed size first row index unencoded bytes
18+
page-0 146 203 0 -
19+
20+
Column index for column test-value
21+
Boundary order: ASCENDING
22+
null count min max rep level histogram def level histogram
23+
page-0 0 10 1000 <none> <none>
24+
25+
Page index for column test-value
26+
offset compressed size first row index unencoded bytes
27+
page-0 349 214 0 -
28+

‎java/clients/src/test/java/sleeper/clients/util/ShowPageIndexes-v2-multiplePages.approved.txt‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
Row group 0
2+
Column index for column test-key
3+
Boundary order: ASCENDING
4+
null count min max rep level histogram def level histogram
5+
page-0 0 row-0001 row-0100 <none> <none>
6+
page-1 0 row-0101 row-0200 <none> <none>
7+
page-2 0 row-0201 row-0300 <none> <none>
8+
page-3 0 row-0301 row-0400 <none> <none>
9+
page-4 0 row-0401 row-0500 <none> <none>
10+
page-5 0 row-0501 row-0600 <none> <none>
11+
page-6 0 row-0601 row-0700 <none> <none>
12+
page-7 0 row-0701 row-0800 <none> <none>
13+
page-8 0 row-0801 row-0900 <none> <none>
14+
page-9 0 row-0901 row-1000 <none> <none>
15+
16+
Page index for column test-key
17+
offset compressed size first row index unencoded bytes
18+
page-0 4 146 0 800
19+
page-1 150 145 100 800
20+
page-2 295 146 200 800
21+
page-3 441 146 300 800
22+
page-4 587 146 400 800
23+
page-5 733 146 500 800
24+
page-6 879 146 600 800
25+
page-7 1025 146 700 800
26+
page-8 1171 144 800 800
27+
page-9 1315 158 900 800
28+

0 commit comments

Comments
 (0)