Handle hardcoded __HIVE_DEFAULT_PARTTION__ in common/FileUtils

Author: vikramahuja1001

common/src/java/org/apache/hadoop/hive/common/FileUtils.java

@@ -64,6 +64,7 @@
 import org.apache.hadoop.fs.PathIsDirectoryException;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
 import org.apache.hadoop.hive.shims.HadoopShims;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.hive.shims.Utils;
@@ -161,9 +162,9 @@ private FileUtils() {
     // prevent instantiation
   }
 
-
-  public static String makePartName(List<String> partCols, List<String> vals) {
-    return makePartName(partCols, vals, null);
+  public static String makePartName(List<String> partCols, List<String> vals, Map<String, String> tableParams,
+      Configuration conf) {
+    return makePartName(partCols, vals, null, tableParams, conf);
   }
 
   /**
@@ -175,15 +176,15 @@ public static String makePartName(List<String> partCols, List<String> vals) {
    * @return An escaped, valid partition name.
    */
   public static String makePartName(List<String> partCols, List<String> vals,
-      String defaultStr) {
+      String defaultStr, Map<String, String> tableParams, Configuration conf) {
     StringBuilder name = new StringBuilder();
     for (int i = 0; i < partCols.size(); i++) {
       if (i > 0) {
         name.append(Path.SEPARATOR);
       }
-      name.append(escapePathName((partCols.get(i)).toLowerCase(), defaultStr));
+      name.append(escapePathName((partCols.get(i)).toLowerCase(), defaultStr, tableParams, conf));
       name.append('=');
-      name.append(escapePathName(vals.get(i), defaultStr));
+      name.append(escapePathName(vals.get(i), defaultStr, tableParams, conf));
     }
     return name.toString();
   }
@@ -196,9 +197,9 @@ public static String makePartName(List<String> partCols, List<String> vals,
    * @return
    */
   public static String makeDefaultListBucketingDirName(List<String> skewedCols,
-      String name) {
+      String name, Map<String, String> tableParams, Configuration conf) {
     String lbDirName;
-    String defaultDir = FileUtils.escapePathName(name);
+    String defaultDir = FileUtils.escapePathName(name, tableParams, conf);
     StringBuilder defaultDirPath = new StringBuilder();
     for (int i = 0; i < skewedCols.size(); i++) {
       if (i > 0) {
@@ -216,15 +217,16 @@ public static String makeDefaultListBucketingDirName(List<String> skewedCols,
    * @param vals The skewed values
    * @return An escaped, valid list bucketing directory name.
    */
-  public static String makeListBucketingDirName(List<String> lbCols, List<String> vals) {
+  public static String makeListBucketingDirName(List<String> lbCols, List<String> vals, Map<String, String> tableParams,
+      Configuration conf) {
     StringBuilder name = new StringBuilder();
     for (int i = 0; i < lbCols.size(); i++) {
       if (i > 0) {
         name.append(Path.SEPARATOR);
       }
-      name.append(escapePathName((lbCols.get(i)).toLowerCase()));
+      name.append(escapePathName((lbCols.get(i)).toLowerCase(), tableParams, conf));
       name.append('=');
-      name.append(escapePathName(vals.get(i)));
+      name.append(escapePathName(vals.get(i), tableParams, conf));
     }
     return name.toString();
   }
@@ -276,8 +278,8 @@ static boolean needsEscaping(char c) {
     return c < charToEscape.size() && charToEscape.get(c);
   }
 
-  public static String escapePathName(String path) {
-    return escapePathName(path, null);
+  public static String escapePathName(String path, Map<String, String> tableParams, Configuration conf) {
+    return escapePathName(path, null, tableParams, conf);
   }
 
   /**
@@ -287,15 +289,16 @@ public static String escapePathName(String path) {
    *          The default name for the path, if the given path is empty or null.
    * @return An escaped path name.
    */
-  public static String escapePathName(String path, String defaultPath) {
+  public static String escapePathName(String path, String defaultPath, Map<String, String> tableParams,
+      Configuration conf) {
 
     // __HIVE_DEFAULT_NULL__ is the system default value for null and empty string.
     // TODO: we should allow user to specify default partition or HDFS file location.
     if (path == null || path.length() == 0) {
       if (defaultPath == null) {
         //previously, when path is empty or null and no default path is specified,
         // __HIVE_DEFAULT_PARTITION__ was the return value for escapePathName
-        return "__HIVE_DEFAULT_PARTITION__";
+        return MetaStoreUtils.getDefaultPartitionName(tableParams, conf);
       } else {
         return defaultPath;
       }

common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java

@@ -309,7 +309,7 @@ public void testPathEscapeChars() {
     StringBuilder sb = new StringBuilder();
     FileUtils.charToEscape.stream().forEach(integer -> sb.append((char) integer));
     String path = sb.toString();
-    assertEquals(path, FileUtils.unescapePathName(FileUtils.escapePathName(path)));
+    assertEquals(path, FileUtils.unescapePathName(FileUtils.escapePathName(path, null, new HiveConf())));
   }
 
   @Test

hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java

@@ -396,7 +396,8 @@ private Partition constructPartition(
           HdfsUtils.setFullFileStatus(conf, status, status.getFileStatus().getGroup(), fs,
               partPath, false);
         }
-        partPath = constructPartialPartPath(partPath, partKey.getName().toLowerCase(), partKVs);
+        partPath = constructPartialPartPath(partPath, partKey.getName().toLowerCase(), partKVs, table.getParameters(),
+            conf);
       }
     }
 
@@ -428,7 +429,8 @@ private String getFinalDynamicPartitionDestination(Table table, Map<String, Stri
       // file:///tmp/hcat_junit_warehouse/employee/_DYN0.7770480401313761/emp_country=IN/emp_state=KA  ->
       // file:///tmp/hcat_junit_warehouse/employee/emp_country=IN/emp_state=KA
       for (FieldSchema partKey : table.getPartitionKeys()) {
-        partPath = constructPartialPartPath(partPath, partKey.getName().toLowerCase(), partKVs);
+        partPath = constructPartialPartPath(partPath, partKey.getName().toLowerCase(), partKVs, table.getParameters(),
+            conf);
       }
 
       return partPath.toString();
@@ -455,11 +457,12 @@ private Map<String, String> getStorerParameterMap(StorerInfo storer) {
     return params;
   }
 
-  private Path constructPartialPartPath(Path partialPath, String partKey, Map<String, String> partKVs) {
+  private Path constructPartialPartPath(Path partialPath, String partKey, Map<String, String> partKVs,
+      Map<String, String> tableParams, Configuration conf) {
 
-    StringBuilder sb = new StringBuilder(FileUtils.escapePathName(partKey));
+    StringBuilder sb = new StringBuilder(FileUtils.escapePathName(partKey, tableParams, conf));
     sb.append("=");
-    sb.append(FileUtils.escapePathName(partKVs.get(partKey)));
+    sb.append(FileUtils.escapePathName(partKVs.get(partKey), tableParams, conf));
     return new Path(partialPath, sb.toString());
   }
 

hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FosterStorageHandler.java

@@ -184,7 +184,9 @@ public void configureOutputJobProperties(TableDesc tableDesc,
           cols.add(name);
           values.add(value);
         }
-        outputLocation = FileUtils.makePartName(cols, values);
+        outputLocation = FileUtils.makePartName(cols, values, (jobInfo.getTableInfo() != null &&
+            jobInfo.getTableInfo().getTable() != null) ? jobInfo.getTableInfo().getTable().getParameters() : null,
+            conf);
       }
 
       if (outputLocation!= null && !outputLocation.isEmpty()){

ql/src/java/org/apache/hadoop/hive/ql/ddl/table/lock/show/ShowLocksOperation.java

@@ -44,6 +44,7 @@
 import org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement;
 import org.apache.hadoop.hive.ql.Context;
 import org.apache.hadoop.hive.ql.ddl.DDLOperation;
+import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.session.SessionState;
 
@@ -175,7 +176,8 @@ private ShowLocksResponse getLocksForNewFormat(HiveLockManager lockMgr) throws H
         keyList.add(partKey);
         valList.add(partVal);
       }
-      String partName = FileUtils.makePartName(keyList, valList);
+      String partName = FileUtils.makePartName(keyList, valList, Hive.get(context.getConf()).getTable(
+          request.getDbname(), request.getTablename()).getParameters(), context.getConf());
       request.setPartname(partName);
     }
 

ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/truncate/TruncateTableDesc.java

@@ -76,6 +76,10 @@ public String getTableName() {
     return tableName.getNotEmptyDbTable();
   }
 
+  public TableName getTableNameObject() {
+    return tableName;
+  }
+
   @Override
   public String getFullTableName() {
     return tableName.getNotEmptyDbTable();

ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/truncate/TruncateTableOperation.java

@@ -70,7 +70,9 @@ public int execute() throws HiveException {
       // or the existing table is newer than our update.
       LOG.debug("DDLTask: Truncate Table/Partition is skipped as table {} / partition {} is newer than update",
           tableName, (partSpec == null) ?
-              "null" : FileUtils.makePartName(new ArrayList<>(partSpec.keySet()), new ArrayList<>(partSpec.values())));
+              "null" : FileUtils.makePartName(new ArrayList<>(partSpec.keySet()), new ArrayList<>(partSpec.values()),
+                  context.getDb().getTable(desc.getTableNameObject().getDb(), desc.getTableNameObject().getTable()).
+                      getParameters(), context.getConf()));
       return 0;
     }
 

ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/rename/AlterTableRenamePartitionOperation.java

@@ -49,11 +49,13 @@ public int execute() throws HiveException {
     Map<String, String> oldPartSpec = desc.getOldPartSpec();
     ReplicationSpec replicationSpec = desc.getReplicationSpec();
 
+    Table tbl = context.getDb().getTable(tableName);
     if (!AlterTableUtils.allowOperationInReplicationScope(context.getDb(), tableName, oldPartSpec, replicationSpec)) {
       // no rename, the table is missing either due to drop/rename which follows the current rename.
       // or the existing table is newer than our update.
       LOG.debug("DDLTask: Rename Partition is skipped as table {} / partition {} is newer than update", tableName,
-              FileUtils.makePartName(new ArrayList<>(oldPartSpec.keySet()), new ArrayList<>(oldPartSpec.values())));
+              FileUtils.makePartName(new ArrayList<>(oldPartSpec.keySet()), new ArrayList<>(oldPartSpec.values()),
+                  tbl.getParameters(), context.getConf()));
       return 0;
     }
 
@@ -62,11 +64,10 @@ public int execute() throws HiveException {
       throw new HiveException("Rename Partition: Not allowed as bootstrap dump in progress");
     }
 
-    Table tbl = context.getDb().getTable(tableName);
     Partition oldPart = context.getDb().getPartition(tbl, oldPartSpec, false);
     if (oldPart == null) {
       String partName = FileUtils.makePartName(new ArrayList<String>(oldPartSpec.keySet()),
-          new ArrayList<String>(oldPartSpec.values()));
+          new ArrayList<String>(oldPartSpec.values()), tbl.getParameters(), context.getConf());
       throw new HiveException("Rename partition: source partition [" + partName + "] does not exist.");
     }
 

ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java

@@ -1371,7 +1371,8 @@ protected String generateListBucketingDirName(Object row) {
       /* The row matches skewed column names. */
       if (allSkewedVals.contains(skewedValsCandidate)) {
         /* matches skewed values. */
-        lbDirName = FileUtils.makeListBucketingDirName(skewedCols, skewedValsCandidate);
+        lbDirName = FileUtils.makeListBucketingDirName(skewedCols, skewedValsCandidate, conf.getTable() != null ?
+            conf.getTable().getParameters() : null, hconf);
         locationMap.put(skewedValsCandidate, lbDirName);
       } else {
         lbDirName = createDefaultLbDir(skewedCols, locationMap);
@@ -1386,7 +1387,7 @@ private String createDefaultLbDir(List<String> skewedCols,
       Map<List<String>, String> locationMap) {
     String lbDirName;
     lbDirName = FileUtils.makeDefaultListBucketingDirName(skewedCols,
-          lbCtx.getDefaultDirName());
+          lbCtx.getDefaultDirName(), conf.getTable() != null ? conf.getTable().getParameters() : null, hconf);
     List<String> defaultKey = Lists.newArrayList(lbCtx.getDefaultKey());
     if (!locationMap.containsKey(defaultKey)) {
       locationMap.put(defaultKey, lbDirName);
@@ -1474,7 +1475,8 @@ protected FSPaths getDynOutPaths(List<String> row, String lbDir) throws HiveExce
   // return the relative path corresponding to the row.
   // e.g., ds=2008-04-08/hr=11
   private String getDynPartDirectory(List<String> row, List<String> dpColNames) {
-    return FileUtils.makePartName(dpColNames, row);
+    return FileUtils.makePartName(dpColNames, row, conf.getTable() != null ?
+        conf.getTable().getParameters() : null, hconf);
   }
 
   @Override

ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java

@@ -295,7 +295,7 @@ protected void flushToFile() throws IOException, HiveException {
       }
       // get current input file name
       String bigBucketFileName = getExecContext().getCurrentBigBucketFile();
-      String fileName = getExecContext().getLocalWork().getBucketFileName(bigBucketFileName);
+      String fileName = getExecContext().getLocalWork().getBucketFileName(bigBucketFileName, hconf);
       // get the tmp URI path; it will be a hdfs path if not local mode
       // TODO [MM gap?]: this doesn't work, however this is MR only.
       //      The path for writer and reader mismatch: