HIVE-28520: Upgrade to datasketches 2.0.0 (#5444)

Author: zhangbutao

pom.xml

@@ -220,7 +220,7 @@
     <rs-api.version>2.0.1</rs-api.version>
     <json-path.version>2.9.0</json-path.version>
     <janino.version>3.1.12</janino.version>
-    <datasketches.version>1.2.0</datasketches.version>
+    <datasketches.version>2.0.0</datasketches.version>
     <spotbugs.version>4.8.6</spotbugs.version>
     <validation-api.version>1.1.0.Final</validation-api.version>
     <aws-secretsmanager-caching.version>1.0.1</aws-secretsmanager-caching.version>

ql/src/java/org/apache/hadoop/hive/ql/ddl/ShowUtils.java

@@ -23,6 +23,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.datasketches.kll.KllFloatsSketch;
 import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.quantilescommon.QuantileSearchCriteria;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -257,7 +258,7 @@ private static String convertHistogram(byte[] buffer, ColumnStatisticsData._Fiel
     final KllFloatsSketch kll = KllFloatsSketch.heapify(Memory.wrap(buffer));
     // to keep the visualization compact, we print only the quartiles (Q1, Q2 and Q3),
     // as min and max are displayed as separate statistics already
-    final float[] quantiles = kll.getQuantiles(new double[]{ 0.25, 0.5, 0.75 });
+    final float[] quantiles = kll.getQuantiles(new double[]{ 0.25, 0.5, 0.75 },  QuantileSearchCriteria.EXCLUSIVE);
 
     Function<Float, Object> converter;
 

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java

@@ -42,6 +42,7 @@
 import org.apache.calcite.util.ImmutableBitSet;
 import org.apache.datasketches.kll.KllFloatsSketch;
 import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.quantilescommon.QuantileSearchCriteria;
 import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
 import org.apache.hadoop.hive.ql.optimizer.calcite.HiveConfPlannerContext;
 import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
@@ -489,7 +490,7 @@ public Double visitLiteral(RexLiteral literal) {
 
   private static double rangedSelectivity(KllFloatsSketch kll, float val1, float val2) {
     float[] splitPoints = new float[] { val1, val2 };
-    double[] boundaries = kll.getCDF(splitPoints);
+    double[] boundaries = kll.getCDF(splitPoints, QuantileSearchCriteria.EXCLUSIVE);
     return boundaries[1] - boundaries[0];
   }
 
@@ -500,7 +501,7 @@ private static double rangedSelectivity(KllFloatsSketch kll, float val1, float v
    * @return the selectivity of a predicate "column > value" in the range [0, 1]
    */
   public static double greaterThanSelectivity(KllFloatsSketch kll, float value) {
-    float max = kll.getMaxValue();
+    float max = kll.getMaxItem();
     if (value > max) {
       return 0;
     }
@@ -518,10 +519,10 @@ public static double greaterThanSelectivity(KllFloatsSketch kll, float value) {
    * @return the selectivity of a predicate "column >= value" in the range [0, 1]
    */
   public static double greaterThanOrEqualSelectivity(KllFloatsSketch kll, float value) {
-    if (value > kll.getMaxValue()) {
+    if (value > kll.getMaxItem()) {
       return 0;
     }
-    return rangedSelectivity(kll, value, Math.nextUp(kll.getMaxValue()));
+    return rangedSelectivity(kll, value, Math.nextUp(kll.getMaxItem()));
   }
 
   /**
@@ -531,10 +532,10 @@ public static double greaterThanOrEqualSelectivity(KllFloatsSketch kll, float va
    * @return the selectivity of a predicate "column <= value" in the range [0, 1]
    */
   public static double lessThanOrEqualSelectivity(KllFloatsSketch kll, float value) {
-    if (value < kll.getMinValue()) {
+    if (value < kll.getMinItem()) {
       return 0;
     }
-    return kll.getCDF(new float[] { Math.nextUp(value) })[0];
+    return kll.getCDF(new float[] { Math.nextUp(value) }, QuantileSearchCriteria.EXCLUSIVE)[0];
   }
 
   /**
@@ -544,14 +545,14 @@ public static double lessThanOrEqualSelectivity(KllFloatsSketch kll, float value
    * @return the selectivity of a predicate "column &lt; value" in the range [0, 1]
    */
   public static double lessThanSelectivity(KllFloatsSketch kll, float value) {
-    float min = kll.getMinValue();
+    float min = kll.getMinItem();
     if (value < min) {
       return 0;
     }
     if (Double.compare(value, min) == 0 || Double.compare(Math.nextUp(value), min) == 0) {
       return 0;
     }
-    return kll.getCDF(new float[] { value })[0];
+    return kll.getCDF(new float[] { value }, QuantileSearchCriteria.EXCLUSIVE)[0];
   }
 
   /**

ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java

@@ -37,7 +37,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import org.apache.calcite.rel.metadata.RelMdUtil;
-import org.apache.datasketches.SketchesArgumentException;
+import org.apache.datasketches.common.SketchesArgumentException;
 import org.apache.datasketches.kll.KllFloatsSketch;
 import org.apache.datasketches.memory.Memory;
 import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;