Skip to content

Added support for the timestamp type for statistics in trino-hive connector #26338

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Added support for the timestamp type for statistics in trino-hive connector #26338

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 44 additions & 3 deletions ...n/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftMetastoreUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@
import io.trino.hive.thrift.metastore.SerDeInfo;
import io.trino.hive.thrift.metastore.StorageDescriptor;
import io.trino.hive.thrift.metastore.StringColumnStatsData;
import io.trino.hive.thrift.metastore.Timestamp;
import io.trino.hive.thrift.metastore.TimestampColumnStatsData;
import io.trino.metastore.AcidOperation;
import io.trino.metastore.Column;
import io.trino.metastore.Database;
Expand Down Expand Up @@ -83,6 +85,8 @@
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.ArrayDeque;
import java.util.Arrays;
import java.util.Collection;
Expand Down Expand Up @@ -148,8 +152,13 @@
import static io.trino.spi.type.IntegerType.INTEGER;
import static io.trino.spi.type.RealType.REAL;
import static io.trino.spi.type.SmallintType.SMALLINT;
import static io.trino.spi.type.StandardTypes.TIMESTAMP;
import static io.trino.spi.type.Timestamps.MICROSECONDS_PER_MILLISECOND;
import static io.trino.spi.type.Timestamps.MICROSECONDS_PER_SECOND;
import static io.trino.spi.type.TinyintType.TINYINT;
import static io.trino.spi.type.VarbinaryType.VARBINARY;
import static java.lang.Math.ceilDiv;
import static java.lang.Math.floorDiv;
import static java.lang.Math.toIntExact;
import static java.lang.String.format;
import static java.util.Locale.ENGLISH;
Expand Down Expand Up @@ -521,6 +530,10 @@ public static HiveColumnStatistics fromMetastoreApiColumnStatistics(ColumnStatis
LongColumnStatsData longStatsData = columnStatistics.getStatsData().getLongStats();
OptionalLong min = longStatsData.isSetLowValue() ? OptionalLong.of(longStatsData.getLowValue()) : OptionalLong.empty();
OptionalLong max = longStatsData.isSetHighValue() ? OptionalLong.of(longStatsData.getHighValue()) : OptionalLong.empty();
if (min.isPresent() && max.isPresent() && columnStatistics.getColType().equals(TIMESTAMP)) {
min = OptionalLong.of(min.getAsLong() * MICROSECONDS_PER_SECOND);
max = OptionalLong.of(max.getAsLong() * MICROSECONDS_PER_SECOND);
}
OptionalLong nullsCount = longStatsData.isSetNumNulls() ? fromMetastoreNullsCount(longStatsData.getNumNulls()) : OptionalLong.empty();
OptionalLong distinctValuesWithNullCount = longStatsData.isSetNumDVs() ? OptionalLong.of(longStatsData.getNumDVs()) : OptionalLong.empty();
return createIntegerColumnStatistics(min, max, nullsCount, distinctValuesWithNullCount);
Expand Down Expand Up @@ -585,6 +598,14 @@ public static HiveColumnStatistics fromMetastoreApiColumnStatistics(ColumnStatis
averageColumnLength,
nullsCount);
}
if (columnStatistics.getStatsData().isSetTimestampStats()) {
TimestampColumnStatsData timestampStatsData = columnStatistics.getStatsData().getTimestampStats();
OptionalLong min = timestampStatsData.isSetLowValue() ? fromMetastoreTimestamp(timestampStatsData.getLowValue()) : OptionalLong.empty();
OptionalLong max = timestampStatsData.isSetHighValue() ? fromMetastoreTimestamp(timestampStatsData.getHighValue()) : OptionalLong.empty();
OptionalLong nullsCount = timestampStatsData.isSetNumNulls() ? fromMetastoreNullsCount(timestampStatsData.getNumNulls()) : OptionalLong.empty();
OptionalLong distinctValuesWithNullCount = timestampStatsData.isSetNumDVs() ? OptionalLong.of(timestampStatsData.getNumDVs()) : OptionalLong.empty();
return createIntegerColumnStatistics(min, max, nullsCount, distinctValuesWithNullCount);
}
throw new TrinoException(HIVE_INVALID_METADATA, "Invalid column statistics data: " + columnStatistics);
}

Expand All @@ -609,6 +630,14 @@ public static OptionalLong fromMetastoreNullsCount(long nullsCount)
return OptionalLong.of(nullsCount);
}

private static OptionalLong fromMetastoreTimestamp(Timestamp timestamp)
{
if (timestamp == null) {
return OptionalLong.empty();
}
return OptionalLong.of(LocalDateTime.ofEpochSecond(timestamp.getSecondsSinceEpoch(), 0, ZoneOffset.UTC).toInstant(ZoneOffset.UTC).toEpochMilli() * MICROSECONDS_PER_MILLISECOND);
}

private static Optional<BigDecimal> fromMetastoreDecimal(@Nullable Decimal decimal)
{
if (decimal == null) {
Expand Down Expand Up @@ -768,8 +797,9 @@ public static ColumnStatisticsObj createMetastoreColumnStatistics(String columnN
case SHORT:
case INT:
case LONG:
case TIMESTAMP:
return createLongStatistics(columnName, columnType, statistics);
case TIMESTAMP:
return createTimestampStatistics(columnName, columnType, statistics);
case FLOAT:
case DOUBLE:
return createDoubleStatistics(columnName, columnType, statistics);
Expand Down Expand Up @@ -819,6 +849,18 @@ private static ColumnStatisticsObj createLongStatistics(String columnName, HiveT
return new ColumnStatisticsObj(columnName, columnType.toString(), longStats(data));
}

private static ColumnStatisticsObj createTimestampStatistics(String columnName, HiveType columnType, HiveColumnStatistics statistics)
{
LongColumnStatsData data = new LongColumnStatsData();
statistics.getIntegerStatistics().ifPresent(timestampStatistics -> {
timestampStatistics.getMin().ifPresent(value -> data.setLowValue(floorDiv(value, MICROSECONDS_PER_SECOND)));
timestampStatistics.getMax().ifPresent(value -> data.setHighValue(ceilDiv(value, MICROSECONDS_PER_SECOND)));
});
statistics.getNullsCount().ifPresent(data::setNumNulls);
statistics.getDistinctValuesWithNullCount().ifPresent(data::setNumDVs);
return new ColumnStatisticsObj(columnName, columnType.toString(), longStats(data));
}

private static ColumnStatisticsObj createDoubleStatistics(String columnName, HiveType columnType, HiveColumnStatistics statistics)
{
DoubleColumnStatsData data = new DoubleColumnStatsData();
Expand Down Expand Up @@ -893,8 +935,7 @@ public static Set<HiveColumnStatisticType> getSupportedColumnStatistics(Type typ
return ImmutableSet.of(MIN_VALUE, MAX_VALUE, NUMBER_OF_DISTINCT_VALUES, NUMBER_OF_NON_NULL_VALUES);
}
if (type instanceof TimestampType || type instanceof TimestampWithTimeZoneType) {
// TODO (https://github.com/trinodb/trino/issues/5859) Add support for timestamp MIN_VALUE, MAX_VALUE
return ImmutableSet.of(NUMBER_OF_DISTINCT_VALUES, NUMBER_OF_NON_NULL_VALUES);
return ImmutableSet.of(MIN_VALUE, MAX_VALUE, NUMBER_OF_DISTINCT_VALUES, NUMBER_OF_NON_NULL_VALUES);
}
if (type instanceof VarcharType || type instanceof CharType) {
// TODO Collect MIN,MAX once it is used by the optimizer
Expand Down
12 changes: 12 additions & 0 deletions ...no-hive/src/main/java/io/trino/plugin/hive/statistics/AbstractHiveStatisticsProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import io.trino.spi.statistics.TableStatistics;
import io.trino.spi.type.CharType;
import io.trino.spi.type.DecimalType;
import io.trino.spi.type.TimestampType;
import io.trino.spi.type.Type;
import io.trino.spi.type.VarcharType;

Expand Down Expand Up @@ -843,6 +844,9 @@ private static Optional<DoubleRange> createRange(Type type, HiveColumnStatistics
if (type.equals(DATE)) {
return statistics.getDateStatistics().flatMap(AbstractHiveStatisticsProvider::createDateRange);
}
if (type instanceof TimestampType) {
return statistics.getIntegerStatistics().flatMap(AbstractHiveStatisticsProvider::createTimestampRange);
}
if (type instanceof DecimalType) {
return statistics.getDecimalStatistics().flatMap(AbstractHiveStatisticsProvider::createDecimalRange);
}
Expand Down Expand Up @@ -895,6 +899,14 @@ private static Optional<DoubleRange> createDateRange(DateStatistics statistics)
return Optional.empty();
}

private static Optional<DoubleRange> createTimestampRange(IntegerStatistics statistics)
{
if (statistics.getMin().isPresent() && statistics.getMax().isPresent()) {
return Optional.of(new DoubleRange(statistics.getMin().getAsLong(), statistics.getMax().getAsLong()));
}
return Optional.empty();
}

private static Optional<DoubleRange> createDecimalRange(DecimalStatistics statistics)
{
if (statistics.getMin().isPresent() && statistics.getMax().isPresent()) {
Expand Down
25 changes: 23 additions & 2 deletions plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/Statistics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,13 @@
import io.trino.spi.Page;
import io.trino.spi.TrinoException;
import io.trino.spi.block.Block;
import io.trino.spi.block.Fixed12Block;
import io.trino.spi.statistics.ColumnStatisticMetadata;
import io.trino.spi.statistics.ComputedStatistics;
import io.trino.spi.type.DecimalType;
import io.trino.spi.type.Decimals;
import io.trino.spi.type.LongTimestamp;
import io.trino.spi.type.TimestampType;
import io.trino.spi.type.Type;

import java.math.BigDecimal;
Expand Down Expand Up @@ -65,6 +68,7 @@
import static io.trino.spi.type.IntegerType.INTEGER;
import static io.trino.spi.type.RealType.REAL;
import static io.trino.spi.type.SmallintType.SMALLINT;
import static io.trino.spi.type.TimestampType.MAX_PRECISION;
import static io.trino.spi.type.TinyintType.TINYINT;
import static java.lang.Float.intBitsToFloat;
import static java.lang.Math.toIntExact;
Expand Down Expand Up @@ -128,10 +132,12 @@ else if (type.equals(DOUBLE) || type.equals(REAL)) {
else if (type.equals(DATE)) {
result.setDateStatistics(new DateStatistics(Optional.empty(), Optional.empty()));
}
else if (type instanceof TimestampType) {
result.setIntegerStatistics(new IntegerStatistics(OptionalLong.empty(), OptionalLong.empty()));
}
else if (type instanceof DecimalType) {
result.setDecimalStatistics(new DecimalStatistics(Optional.empty(), Optional.empty()));
}
// TODO (https://github.com/trinodb/trino/issues/5859) Add support for timestamp
else {
throw new IllegalArgumentException("Unexpected type: " + type);
}
Expand Down Expand Up @@ -240,10 +246,12 @@ else if (type.equals(DOUBLE) || type.equals(REAL)) {
else if (type.equals(DATE)) {
result.setDateStatistics(new DateStatistics(getDateValue(type, min), getDateValue(type, max)));
}
else if (type instanceof TimestampType) {
result.setIntegerStatistics(new IntegerStatistics(getTimestampValue(type, min), getTimestampValue(type, max)));
}
else if (type instanceof DecimalType) {
result.setDecimalStatistics(new DecimalStatistics(getDecimalValue(type, min), getDecimalValue(type, max)));
}
// TODO (https://github.com/trinodb/trino/issues/5859) Add support for timestamp
else {
throw new IllegalArgumentException("Unexpected type: " + type);
}
Expand Down Expand Up @@ -288,6 +296,19 @@ private static Optional<LocalDate> getDateValue(Type type, Block block)
return Optional.of(LocalDate.ofEpochDay(days));
}

private static OptionalLong getTimestampValue(Type type, Block block)
{
verify(type instanceof TimestampType, "Unsupported type: %s", type);
if (block.isNull(0)) {
return OptionalLong.empty();
}
if (block instanceof Fixed12Block) {
LongTimestamp ts = (LongTimestamp) TimestampType.createTimestampType(MAX_PRECISION).getObject(block, 0);
return OptionalLong.of(ts.getEpochMicros());
}
return OptionalLong.of(type.getLong(block, 0));
}

private static Optional<BigDecimal> getDecimalValue(Type type, Block block)
{
verify(type instanceof DecimalType, "Unsupported type: %s", type);
Expand Down
Loading
Loading