Handle hardcoded __HIVE_DEFAULT_PARTTION__ in metastore-common/FileUtils

Author: vikramahuja1001

hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java

@@ -410,7 +410,8 @@ public PartitionFiles next() {
           fileIterator = Collections.emptyIterator();
         }
         PartitionFiles partitionFiles =
-            new PartitionFiles(Warehouse.makePartName(t.getPartitionKeys(), p.getValues()), fileIterator);
+            new PartitionFiles(Warehouse.makePartName(t.getPartitionKeys(), p.getValues(), t.getParameters(), conf),
+                fileIterator);
         return partitionFiles;
       } catch (MetaException e) {
         throw new RuntimeException(e);

hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatPartition.java

@@ -138,7 +138,8 @@ Partition toHivePartition() throws HCatException {
     if (sd.getLocation() == null) {
       LOG.warn("Partition location is not set! Attempting to construct default partition location.");
       try {
-        String partName = Warehouse.makePartName(HCatSchemaUtils.getFieldSchemas(hcatTable.getPartCols()), values);
+        String partName = Warehouse.makePartName(HCatSchemaUtils.getFieldSchemas(hcatTable.getPartCols()), values,
+            hcatTable.getTblProps(), hcatTable.getConf());
         sd.setLocation(new Path(hcatTable.getSd().getLocation(), partName).toString());
       }
       catch(MetaException exception) {

hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java

@@ -156,7 +156,12 @@ public static String fixPath(String path) {
   }
 
   public static String makePartLocation(HCatTable table, Map<String, String> partitionSpec) throws MetaException {
-    return (new Path(table.getSd().getLocation(), Warehouse.makePartPath(partitionSpec))).toUri().toString();
+      try {
+          return (new Path(table.getSd().getLocation(), Warehouse.makePartPath(partitionSpec,
+              table.toHiveTable().getParameters(), getConf()))).toUri().toString();
+      } catch (HCatException e) {
+          throw new RuntimeException(e);
+      }
   }
 
   @Test

iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java

@@ -78,7 +78,6 @@
 import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableDesc;
 import org.apache.hadoop.hive.ql.ddl.table.create.like.CreateTableLikeDesc;
 import org.apache.hadoop.hive.ql.ddl.table.misc.properties.AlterTableSetPropertiesDesc;
-import org.apache.hadoop.hive.ql.ddl.table.partition.PartitionUtils;
 import org.apache.hadoop.hive.ql.exec.ColumnInfo;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.hooks.WriteEntity;
@@ -877,8 +876,7 @@ public DynamicPartitionCtx createDPContext(
     Table table = IcebergTableUtil.getTable(conf, tableDesc.getProperties());
 
     DynamicPartitionCtx dpCtx = new DynamicPartitionCtx(Maps.newLinkedHashMap(),
-        PartitionUtils.getDefaultPartitionName(hmsTable.getParameters(), hiveConf),
-        hiveConf.getIntVar(ConfVars.DYNAMIC_PARTITION_MAX_PARTS_PER_NODE));
+        hiveConf.getIntVar(ConfVars.DYNAMIC_PARTITION_MAX_PARTS_PER_NODE), hmsTable.getParameters(), hiveConf);
 
     if (table.spec().isPartitioned() &&
           hiveConf.getIntVar(ConfVars.HIVE_OPT_SORT_DYNAMIC_PARTITION_THRESHOLD) >= 0) {
@@ -2010,7 +2008,7 @@ public Partition getPartition(org.apache.hadoop.hive.ql.metadata.Table table,
       Map<String, String> partitionSpec, RewritePolicy policy) throws SemanticException {
     validatePartSpec(table, partitionSpec, policy);
     try {
-      String partName = Warehouse.makePartName(partitionSpec, false);
+      String partName = Warehouse.makePartName(partitionSpec, false, table.getParameters(), conf);
       return new DummyPartition(table, partName, partitionSpec);
     } catch (MetaException e) {
       throw new SemanticException("Unable to construct name for dummy partition due to: ", e);

iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java

@@ -124,7 +124,8 @@ public static void importFiles(String sourceLocation,
           Partition partition = partitionIterator.next();
           Callable<Void> task = () -> {
             Path partitionPath = new Path(partition.getSd().getLocation());
-            String partitionName = Warehouse.makePartName(partitionKeys, partition.getValues());
+            String partitionName = Warehouse.makePartName(partitionKeys, partition.getValues(),
+                icebergTable.properties(), conf);
             Map<String, String> partitionSpec = Warehouse.makeSpecFromName(partitionName);
             RemoteIterator<LocatedFileStatus> iterator = getFilesIterator(partitionPath, conf);
             List<DataFile> dataFiles =

itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java

@@ -197,15 +197,15 @@ public void testRecyclePartTable() throws Exception {
     Partition part3 = createPartition(dbName, tblName, columns, values, serdeInfo);
     client.add_partition(part3);
 
-    Path part1Path = new Path(warehouse.getDefaultPartitionPath(db, tbl, ImmutableMap.of("dt", "20160101")), "part");
+    Path part1Path = new Path(warehouse.getDefaultPartitionPath(db, tbl, ImmutableMap.of("dt", "20160101"), hiveConf), "part");
     createFile(part1Path, "p1");
     String path1Chksum = ReplChangeManager.checksumFor(part1Path, fs);
 
-    Path part2Path = new Path(warehouse.getDefaultPartitionPath(db, tbl, ImmutableMap.of("dt", "20160102")), "part");
+    Path part2Path = new Path(warehouse.getDefaultPartitionPath(db, tbl, ImmutableMap.of("dt", "20160102"), hiveConf), "part");
     createFile(part2Path, "p2");
     String path2Chksum = ReplChangeManager.checksumFor(part2Path, fs);
 
-    Path part3Path = new Path(warehouse.getDefaultPartitionPath(db, tbl, ImmutableMap.of("dt", "20160103")), "part");
+    Path part3Path = new Path(warehouse.getDefaultPartitionPath(db, tbl, ImmutableMap.of("dt", "20160103"), hiveConf), "part");
     createFile(part3Path, "p3");
     String path3Chksum = ReplChangeManager.checksumFor(part3Path, fs);
 

itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java

@@ -4320,7 +4320,7 @@ public void testAcidTablesBootstrapWithMajorCompaction() throws Throwable {
     List<Partition> partList = primary.getAllPartitions(primaryDbName, tableNamepart);
     for (Partition part : partList) {
       Table tbl = primary.getTable(primaryDbName, tableNamepart);
-      String partName = Warehouse.makePartName(tbl.getPartitionKeys(), part.getValues());
+      String partName = Warehouse.makePartName(tbl.getPartitionKeys(), part.getValues(), tbl.getParameters(), conf);
       runCompaction(primaryDbName, tableNamepart, partName, CompactionType.MAJOR);
     }
 
@@ -4377,7 +4377,7 @@ public void testAcidTablesBootstrapWithMinorCompaction() throws Throwable {
     List<Partition> partList = primary.getAllPartitions(primaryDbName, tableNamepart);
     for (Partition part : partList) {
       Table tbl = primary.getTable(primaryDbName, tableNamepart);
-      String partName = Warehouse.makePartName(tbl.getPartitionKeys(), part.getValues());
+      String partName = Warehouse.makePartName(tbl.getPartitionKeys(), part.getValues(), tbl.getParameters(), conf);
       runCompaction(primaryDbName, tableNamepart, partName, CompactionType.MINOR);
     }
     List<String> withClause = Arrays.asList(

itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java

@@ -191,14 +191,15 @@ private void verifyReplicatedStatsForPartitionsOfTable(String tableName)
       return;
     }
 
-    List<FieldSchema> partKeys = primary.getTable(primaryDbName, tableName).getPartitionKeys();
+    Table table = primary.getTable(primaryDbName, tableName);
+    List<FieldSchema> partKeys = table.getPartitionKeys();
     for (Partition pPart : pParts) {
       Partition rPart = replica.getPartition(replicatedDbName, tableName,
               pPart.getValues());
 
       Map<String, String> rParams = collectStatsParams(rPart.getParameters());
       Map<String, String> pParams = collectStatsParams(pPart.getParameters());
-      String partName = Warehouse.makePartName(partKeys, pPart.getValues());
+      String partName = Warehouse.makePartName(partKeys, pPart.getValues(), table.getParameters(), conf);
       Assert.assertEquals("Mismatch in stats parameters for partition " + partName + " of table " + tableName,
                           pParams, rParams);
 

ql/src/java/org/apache/hadoop/hive/ql/ddl/process/show/compactions/ShowCompactionsOperation.java

@@ -97,7 +97,12 @@ private ShowCompactRequest getShowCompactioRequest(ShowCompactionsDesc desc) thr
       request.setState(compactionStateStr2Enum(desc.getCompactionStatus()).getSqlConst());
     }
     if (isNotEmpty(desc.getPartSpec())) {
-      request.setPartName(AcidUtils.getPartitionName(desc.getPartSpec()));
+        try {
+            request.setPartName(AcidUtils.getPartitionName(desc.getPartSpec(), context.getDb().getTable(
+                request.getDbName(), request.getTbName()).getParameters(), context.getConf()));
+        } catch (HiveException e) {
+            throw new RuntimeException(e);
+        }
     }
     if(desc.getCompactionId()>0){
      request.setId(desc.getCompactionId());

ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/columnstats/AlterTableUpdateColumnStatistictAnalyzer.java

@@ -51,7 +51,7 @@ protected void analyzeCommand(TableName tableName, Map<String, String> partition
     String columnName = getUnescapedName((ASTNode) command.getChild(0));
     Map<String, String> properties = getProps((ASTNode) (command.getChild(1)).getChild(0));
 
-    String partitionName = AcidUtils.getPartitionName(partitionSpec);
+    String partitionName = AcidUtils.getPartitionName(partitionSpec, table.getParameters(), conf);
     String columnType = getColumnType(table, columnName);
 
     ColumnStatsUpdateWork work = new ColumnStatsUpdateWork(partitionName, properties, table.getDbName(),