Ignore authority when comparing hopsfs paths

Author: gibchikafa

hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java

@@ -726,12 +726,26 @@ public static List<StoragePathInfo> getAllDataPathInfo(HoodieStorage storage, St
     }
     return pathInfoList;
   }
-
+  
   public static boolean comparePathsWithoutScheme(String pathStr1, String pathStr2) {
-    Path pathWithoutScheme1 = getPathWithoutScheme(new Path(pathStr1));
-    Path pathWithoutScheme2 = getPathWithoutScheme(new Path(pathStr2));
+    // In hopsfs the following the authority does not matter, so we just compare the paths without scheme and authority.
+    // The following paths are equivalent: hopsfs://10.244.231.90:8020/apps/hive/warehouse/g1_featurestore.db/loans_1,
+    // hopsfs://rpc.namenode.service.consul:8020/apps/hive/warehouse/g1_featurestore.db/loans_1.
+    Path path1 = new Path(pathStr1);
+    Path path2 = new Path(pathStr2);
+    Path pathWithoutScheme1 =
+        path1.isUriPathAbsolute() && path1.toUri().getScheme().equals("hopsfs")
+            ? getPathWithoutSchemeAndAuthority(path1) : getPathWithoutScheme(new Path(pathStr1));
+    Path pathWithoutScheme2 =
+        path2.isUriPathAbsolute() && path2.toUri().getScheme().equals("hopsfs")
+            ? getPathWithoutSchemeAndAuthority(path2) : getPathWithoutScheme(new Path(pathStr2));
     return pathWithoutScheme1.equals(pathWithoutScheme2);
   }
+  
+  public static Path getPathWithoutSchemeAndAuthority(Path path) {
+    return path.isUriPathAbsolute()
+        ? new Path(path.toUri().getPath()) : path;
+  }
 
   public static Path getPathWithoutScheme(Path path) {
     return path.isUriPathAbsolute()

hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncTool.java

@@ -27,8 +27,6 @@
 import org.apache.hadoop.fs.FileSystem;
 
 import java.util.Properties;
-import java.util.logging.Level;
-import java.util.logging.Logger;
 
 import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_BASE_PATH;
 
@@ -37,7 +35,6 @@
  * Hudi table queryable through external systems.
  */
 public abstract class HoodieSyncTool implements AutoCloseable {
-  private final Logger logger = Logger.getLogger(HoodieSyncTool.class.getName());
   protected Properties props;
   protected Configuration hadoopConf;
   protected HoodieMetaSyncMetrics metrics;
@@ -49,9 +46,7 @@ public HoodieSyncTool(Properties props) {
   public HoodieSyncTool(Properties props, Configuration hadoopConf) {
     this.props = props;
     this.hadoopConf = hadoopConf;
-    HoodieSyncConfig hoodieSyncConfig = new HoodieSyncConfig(props, hadoopConf);
-    logger.log(Level.INFO, "Initializing HoodieSyncTool with config: " + hoodieSyncConfig.toString());
-    this.metrics = new HoodieMetaSyncMetrics(hoodieSyncConfig, getClass().getSimpleName());
+    this.metrics = new HoodieMetaSyncMetrics(new HoodieSyncConfig(props, hadoopConf), getClass().getSimpleName());
   }
 
   @Deprecated

pom.xml

@@ -99,7 +99,7 @@
     <fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version>
     <fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version>
     <fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}</fasterxml.jackson.dataformat.yaml.version>
-    <kafka.version>2.6.0</kafka.version>
+    <kafka.version>3.4.1</kafka.version>
     <pulsar.version>3.0.2</pulsar.version>
     <kafka.connect.api.version>2.5.0</kafka.connect.api.version>
     <kafka.spark3.version>2.8.2</kafka.spark3.version>
@@ -2706,7 +2706,7 @@
         <!-- This glob has to include hudi-spark3-common, hudi-spark3.2plus-common -->
         <hudi.spark.common.module>hudi-spark3-common</hudi.spark.common.module>
         <scalatest.version>${scalatest.spark3.version}</scalatest.version>
-        <kafka.version>2.6.0</kafka.version>
+        <kafka.version>3.4.1</kafka.version>
         <hive.storage.version>2.6.1.3</hive.storage.version>
         <!-- NOTE: Some Hudi modules require standalone Parquet/Orc/etc file-format dependency (hudi-hive-sync,
                    hudi-hadoop-mr, for ex). Since these Hudi modules might be used from w/in the execution engine(s)