HIVE-1996: Beeline performance poor with drivers having slow DatabaseMetaData.getPrimaryKeys impl (#6023)

Author: InvisibleProgrammer

beeline/src/java/org/apache/hive/beeline/Rows.java

@@ -29,9 +29,13 @@
 import java.sql.SQLException;
 import java.text.DecimalFormat;
 import java.text.NumberFormat;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Base64;
+import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
 
 import org.apache.hadoop.hive.common.cli.EscapeCRLFHelper;
 
@@ -42,7 +46,8 @@
 abstract class Rows implements Iterator {
   protected final BeeLine beeLine;
   final ResultSetMetaData rsMeta;
-  final Boolean[] primaryKeys;
+  final boolean[] primaryKeyColumns;
+  boolean isPrimaryKeyColumnsInitialized;
   final NumberFormat numberFormat;
   private boolean convertBinaryArrayToString;
   private final String nullStr;
@@ -51,14 +56,15 @@ abstract class Rows implements Iterator {
     this.beeLine = beeLine;
     nullStr = beeLine.getOpts().getNullString();
     rsMeta = rs.getMetaData();
-    int count = rsMeta.getColumnCount();
-    primaryKeys = new Boolean[count];
     if (beeLine.getOpts().getNumberFormat().equals("default")) {
       numberFormat = null;
     } else {
       numberFormat = new DecimalFormat(beeLine.getOpts().getNumberFormat());
     }
     this.convertBinaryArrayToString = beeLine.getOpts().getConvertBinaryArrayToString();
+
+    int count = this.rsMeta.getColumnCount();
+    primaryKeyColumns = new boolean[count];
   }
 
   @Override
@@ -78,37 +84,47 @@ public void remove() {
    * JDBC driver property implements {@link ResultSetMetaData#getTableName} (many do not), it
    * is not reliable for all databases.
    */
-  boolean isPrimaryKey(int col) {
-    if (primaryKeys[col] == null) {
-      try {
-        // this doesn't always work, since some JDBC drivers (e.g.,
-        // Oracle's) return a blank string from getDbTableName.
-        String table = rsMeta.getTableName(col + 1);
-        String column = rsMeta.getColumnName(col + 1);
+  boolean isPrimaryKeyCol(int col) {
+    if (!isPrimaryKeyColumnsInitialized) {
+      initializePrimaryKeyMetadata();
+    }
+
+    return primaryKeyColumns[col];
+  }
+
+  private void initializePrimaryKeyMetadata() {
+    Map<String, List<String>> tablePrimaryKeys = new HashMap<>();
+
+    try {
+      for (int i = 0; i < primaryKeyColumns.length; i++) {
+        String table = rsMeta.getTableName(i + 1);
+        String column = rsMeta.getColumnName(i + 1);
 
         if (table == null || table.isEmpty() || column == null || column.isEmpty()) {
-          primaryKeys[col] = Boolean.FALSE;
-        } else {
-          ResultSet pks = beeLine.getDatabaseConnection().getDatabaseMetaData().getPrimaryKeys(
-              beeLine.getDatabaseConnection().getDatabaseMetaData().getConnection().getCatalog(), null, table);
+          continue;
+        }
+
+        if (!tablePrimaryKeys.containsKey(table)) {
+          try (ResultSet pks = beeLine.getDatabaseConnection().getDatabaseMetaData().getPrimaryKeys(
+                  beeLine.getDatabaseConnection().getDatabaseMetaData().getConnection().getCatalog(), null, table)) {
+
+            List<String> pkNames = new ArrayList<>();
 
-          primaryKeys[col] = Boolean.FALSE;
-          try {
             while (pks.next()) {
-              if (column.equalsIgnoreCase(pks.getString("COLUMN_NAME"))) {
-                primaryKeys[col] = Boolean.TRUE;
-                break;
-              }
+              pkNames.add(pks.getString("COLUMN_NAME"));
             }
-          } finally {
-            pks.close();
+
+            tablePrimaryKeys.put(table, pkNames);
           }
         }
-      } catch (SQLException sqle) {
-        primaryKeys[col] = Boolean.FALSE;
+
+        primaryKeyColumns[i] = tablePrimaryKeys.get(table).contains(column);
       }
+    } catch (SQLException e) {
+      // Do nothing. We cannot decide if the given column is a primary key so we keep it as false
     }
-    return primaryKeys[col].booleanValue();
+
+    isPrimaryKeyColumnsInitialized = true;
   }
 
   class Row {
@@ -134,7 +150,7 @@ class Row {
     }
 
     @Override
-    public String toString(){
+    public String toString() {
       return Arrays.asList(values).toString();
     }
 
@@ -156,17 +172,17 @@ public String toString(){
       } catch (Throwable t) {
       }
 
-       for (int i = 0; i < size; i++) {
+      for (int i = 0; i < size; i++) {
         Object o = rs.getObject(i + 1);
         String value = null;
 
         if (o == null) {
           value = nullStr;
         } else if (o instanceof Number) {
           value = numberFormat != null ? numberFormat.format(o) :
-              o instanceof BigDecimal ? ((BigDecimal)o).toPlainString() : o.toString();
+                  o instanceof BigDecimal ? ((BigDecimal) o).toPlainString() : o.toString();
         } else if (o instanceof byte[]) {
-          value = convertBinaryArrayToString ? new String((byte[])o, StandardCharsets.UTF_8) : Base64.getEncoder().withoutPadding().encodeToString((byte[])o);
+          value = convertBinaryArrayToString ? new String((byte[]) o, StandardCharsets.UTF_8) : Base64.getEncoder().withoutPadding().encodeToString((byte[]) o);
         } else {
           value = rs.getString(i + 1);
         }

beeline/src/java/org/apache/hive/beeline/TableOutputFormat.java

@@ -122,14 +122,14 @@ ColorBuffer getOutputString(Rows rows, Rows.Row row, String delim) {
 
       if (row.isMeta) {
         v = beeLine.getColorBuffer().center(row.values[i], row.sizes[i]);
-        if (beeLine.getOpts().getColor() && rows.isPrimaryKey(i)) {
+        if (beeLine.getOpts().getColor() && rows.isPrimaryKeyCol(i)) {
           buf.cyan(v.getMono());
         } else {
           buf.bold(v.getMono());
         }
       } else {
         v = beeLine.getColorBuffer().pad(row.values[i], row.sizes[i]);
-        if (beeLine.getOpts().getColor() && rows.isPrimaryKey(i)) {
+        if (beeLine.getOpts().getColor() && rows.isPrimaryKeyCol(i)) {
           buf.cyan(v.getMono());
         } else {
           buf.append(v.getMono());

beeline/src/test/org/apache/hive/beeline/BeelineMock.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.beeline;
+
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.List;
+
+public class BeelineMock extends BeeLine {
+  public BeelineMock() {
+    this.allPrintedLines = new ArrayList<>();
+  }
+
+  private String lastPrintedLine;
+  private List<String> allPrintedLines;
+
+  @Override
+  final void output(final ColorBuffer msg, boolean newline, PrintStream out) {
+    lastPrintedLine = msg.getMono();
+    allPrintedLines.add(msg.getColor());
+    super.output(msg, newline, out);
+  }
+
+  public String getLastPrintedLine() {
+    return lastPrintedLine;
+  }
+
+  public List<String> getAllPrintedLines() {
+    return allPrintedLines;
+  }
+}