[HWORKS-2204] Test

.github/workflows/build_hudi_with_hopsfs.yml

@@ -0,0 +1,215 @@
+name: Build Hudi with hopsfs
+
+on:
+  workflow_call:
+    inputs:
+      ref:
+        description: 'The ref to checkout for the spark repo, default is master'
+        required: false
+        type: string
+        default: 'master'
+      jira_tag:
+        description: 'The tag to use for the jira release, default is the version from version.txt'
+        required: false
+        type: string
+        default: 'NOJIRA'
+      runner:
+        description: 'The runner to use, default is ubuntu-latest'
+        required: false
+        type: string
+        default: 'ubuntu-latest'
+      build:
+        description: 'Whether to build Hudi or not, default is true'
+        required: false
+        type: boolean
+        default: true
+    secrets: 
+      NEXUS_HARBOR_PASSWORD:
+        description: 'The password for the Nexus user'
+        required: true
+    outputs:
+      pom_version_no_jira:
+        value: ${{ jobs.build-hudi.outputs.pom_version_no_jira }}
+        description: 'The version from the pom.xml without the jira tag'
+      pom_version:
+        value: ${{ jobs.build-hudi.outputs.pom_version }}
+        description: 'The version from the pom.xml with the jira tag'
+      commit_hash:
+        value: ${{ jobs.build-hudi.outputs.commit_hash }}
+        description: 'The commit hash of the hudi repo that was built'
+      jira_tag:
+        value: ${{ jobs.build-hudi.outputs.jira_tag }}
+        description: 'The jira tag that was used for the build'
+      hudi_utils_jar_name:
+        value: ${{ jobs.build-hudi.outputs.hudi_utils_jar_name }}
+        description: 'The name of the hudi utilities jar that was built and uploaded to nexus'
+      hudi_utils_jar_url:
+        value: ${{ jobs.build-hudi.outputs.hudi_utils_jar_url }}
+        description: 'The url of the hudi utilities jar that was built and uploaded to nexus'
+      hudi_spark_jar_name:
+        value: ${{ jobs.build-hudi.outputs.hudi_spark_jar_name }}
+        description: 'The name of the hudi spark jar that was built and uploaded to nexus'
+      hudi_spark_jar_url:
+        value: ${{ jobs.build-hudi.outputs.hudi_spark_jar_url }}
+        description: 'The url of the hudi spark jar that was built and uploaded to nexus'
+      hopsfs_version:
+        value: ${{ jobs.build-hudi.outputs.hopsfs_version }}
+        description: 'The version of hopsfs that was used to build hudi'
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'The ref to checkout for the spark repo, default is master'
+        required: false
+        type: string
+        default: 'master'
+      jira_tag:
+        description: 'The tag to use for the jira release, default is the version from version.txt'
+        required: false
+        type: string
+        default: 'NOJIRA'
+      runner:
+        description: 'The runner to use, default is ubuntu-latest'
+        required: false
+        type: string
+        default: 'ubuntu-latest'
+      build:
+        description: 'Whether to build Hudi or not, default is true'
+        required: false
+        type: boolean
+        default: true
+
+concurrency:
+  group: build-hudi-${{ github.workflow }}-${{ github.job }}-${{ inputs.jira_tag || 'NOJIRA' }}
+  cancel-in-progress: true
+
+env:
+  # HUDI_REF: ${{ inputs.ref || 'release-1.0.2' }}
+  HUDI_REF: ${{ inputs.ref || 'HWORKS-2204-vatj' }}
+  JIRA_TAG: ${{ inputs.jira_tag || 'NOJIRA' }}
+
+jobs:
+  build-hudi:
+    runs-on: ${{ inputs.runner }}
+    outputs:
+      pom_version_no_jira: ${{ steps.prep_version.outputs.hudi_version_no_jira }}
+      pom_version: ${{ steps.prep_version.outputs.pom_version }}
+      commit_hash: ${{ steps.prep_version.outputs.commit_hash }}
+      jira_tag: ${{ env.JIRA_TAG }}
+      hudi_utils_jar_name: ${{ steps.prep_version.outputs.hudi_utils_jar_name }}
+      hudi_utils_jar_url: ${{ steps.prep_version.outputs.hudi_utils_jar_url }}
+      hudi_spark_jar_name: ${{ steps.prep_version.outputs.hudi_spark_jar_name }}
+      hudi_spark_jar_url: ${{ steps.prep_version.outputs.hudi_spark_jar_url }}
+      hopsfs_version: ${{ steps.prep_version.outputs.hopsfs_version }}
+    steps:
+    - name: Checkout hudi repo
+      uses: actions/checkout@v4
+      with:
+        repository: logicalclocks/hudi
+        ref: ${{ env.HUDI_REF }}
+        path: ${{ github.workspace }}/hudi
+
+    - name: To build or not to build
+      id: to_build_or_not_to_build
+      shell: bash
+      env:
+        BUILD_HUDI: ${{ (github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ','), 'build-hudi')) || inputs.build }}
+      run: |
+        if [[ "${{ env.BUILD_HUDI }}" != "true" ]]; then
+          echo "# :recycle: Not building hudi" >> $GITHUB_STEP_SUMMARY
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            echo "This is a pull request and the 'build-hudi' label is not present." >> $GITHUB_STEP_SUMMARY
+            echo "pull_request_labels=${{ join(github.event.pull_request.labels.*.name, ', ') }}" >> $GITHUB_STEP_SUMMARY
+          elif [[ "${{ inputs.build || 'false'}}" != "true" ]]; then
+            echo "The input 'build' is set to false." >> $GITHUB_STEP_SUMMARY
+          fi
+          echo "BUILD_HUDI=$BUILD_HUDI" >> $GITHUB_OUTPUT
+        else
+          echo "# :white_check_mark: Building hudi" >> $GITHUB_STEP_SUMMARY
+          echo "BUILD_HUDI=$BUILD_HUDI" >> $GITHUB_OUTPUT
+        fi
+
+    - name: Set up .m2 settings.xml
+      shell: bash
+      if: steps.to_build_or_not_to_build.outputs.BUILD_HUDI == 'true'
+      env:
+        M2_HOME: ~/.m2
+      run: |
+        echo "M2_HOME var is $M2_HOME" >> $GITHUB_STEP_SUMMARY
+        mkdir -p ~/.m2
+        echo "<settings><servers>" > ~/.m2/settings.xml
+        echo "<server><id>HopsEE</id><username>${{ vars.NEXUS_HARBOR_USER }}</username><password>${{ secrets.NEXUS_HARBOR_PASSWORD }}</password><configuration></configuration></server>" >> ~/.m2/settings.xml
+        echo "<server><id>HiveEE</id><username>${{ vars.NEXUS_HARBOR_USER }}</username><password>${{ secrets.NEXUS_HARBOR_PASSWORD }}</password><configuration></configuration></server>" >> ~/.m2/settings.xml
+        echo "</servers></settings>" >> ~/.m2/settings.xml
+
+    - name: Cache maven
+      id: cache-maven
+      if: steps.to_build_or_not_to_build.outputs.BUILD_HUDI == 'true'
+      uses: actions/cache@v4
+      with:
+        path: |
+          ~/.m2
+          !~/.m2/settings.xml
+        key: ${{ runner.os }}-maven-hudi-${{ hashFiles('hudi/**/pom.xml') }}
+        restore-keys: |
+          ${{ runner.os }}-maven-hudi-
+
+    - name: Prep step version
+      shell: bash
+      id: prep_version
+      working-directory: ${{ github.workspace }}/hudi
+      run: |
+        COMMIT_HASH=$(git rev-parse --short HEAD)   
+        POM_VERSION_NO_JIRA=$(mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive exec:exec)
+        find . -name "pom.xml" -exec sed -i "s|<version>${POM_VERSION_NO_JIRA}</version>|<version>${POM_VERSION_NO_JIRA}-${JIRA_TAG}-SNAPSHOT</version>|g" {} \;
+        POM_VERSION=$(mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive exec:exec)
+        HUDI_SPARK_JAR_NAME=hudi-spark3.5-bundle_2.12-${POM_VERSION}.jar
+        HUDI_UTILS_JAR_NAME=hudi-utilities-slim-bundle_2.12-${POM_VERSION}.jar
+        HUDI_SPARK_JAR_URL="${{ vars.NEXUS_DEV_SPARK_URL }}/${JIRA_TAG}/${HUDI_SPARK_JAR_NAME}"
+        HUDI_UTILS_JAR_URL="${{ vars.NEXUS_DEV_SPARK_URL }}/${JIRA_TAG}/${HUDI_UTILS_JAR_NAME}"
+        HOPSFS_VERSION=$(mvn -q -Dexec.executable="echo" -Dexec.args='${hadoop.version}' --non-recursive exec:exec)
+
+        echo "POM_VERSION=${POM_VERSION}" >> $GITHUB_ENV
+        echo "POM_VERSION_NO_JIRA=${POM_VERSION_NO_JIRA}" >> $GITHUB_ENV
+        echo "COMMIT_HASH=$COMMIT_HASH" >> $GITHUB_ENV
+        echo "HUDI_SPARK_JAR_NAME=${HUDI_SPARK_JAR_NAME}" >> $GITHUB_ENV
+        echo "HUDI_SPARK_JAR_URL=${HUDI_SPARK_JAR_URL}" >> $GITHUB_ENV
+        echo "HUDI_UTILS_JAR_NAME=${HUDI_UTILS_JAR_NAME}" >> $GITHUB_ENV
+        echo "HUDI_UTILS_JAR_URL=${HUDI_UTILS_JAR_URL}" >> $GITHUB_ENV
+        echo "HOPSFS_VERSION=${HOPSFS_VERSION}" >> $GITHUB_ENV
+
+        echo "POM_VERSION_NO_JIRA=${POM_VERSION_NO_JIRA}" >> $GITHUB_STEP_SUMMARY
+        echo "POM_VERSION=${POM_VERSION}" >> $GITHUB_STEP_SUMMARY
+        echo "COMMIT_HASH=$COMMIT_HASH" >> $GITHUB_STEP_SUMMARY
+        echo "HUDI_SPARK_JAR_NAME=${HUDI_SPARK_JAR_NAME}" >> $GITHUB_STEP_SUMMARY
+        echo "HUDI_SPARK_JAR_URL=${HUDI_SPARK_JAR_URL}" >> $GITHUB_STEP_SUMMARY
+        echo "HUDI_UTILS_JAR_NAME=${HUDI_UTILS_JAR_NAME}" >> $GITHUB_STEP_SUMMARY
+        echo "HUDI_UTILS_JAR_URL=${HUDI_UTILS_JAR_URL}" >> $GITHUB_STEP_SUMMARY
+        echo "HOPSFS_VERSION=${HOPSFS_VERSION}" >> $GITHUB_STEP_SUMMARY
+
+        echo "POM_VERSION_NO_JIRA=${POM_VERSION_NO_JIRA}" >> $GITHUB_OUTPUT
+        echo "POM_VERSION=${POM_VERSION}" >> $GITHUB_OUTPUT
+        echo "COMMIT_HASH=$COMMIT_HASH" >> $GITHUB_OUTPUT
+        echo "HUDI_SPARK_JAR_NAME=${HUDI_SPARK_JAR_NAME}" >> $GITHUB_OUTPUT
+        echo "HUDI_SPARK_JAR_URL=${HUDI_SPARK_JAR_URL}" >> $GITHUB_OUTPUT
+        echo "HUDI_UTILS_JAR_NAME=${HUDI_UTILS_JAR_NAME}" >> $GITHUB_OUTPUT
+        echo "HUDI_UTILS_JAR_URL=${HUDI_UTILS_JAR_URL}" >> $GITHUB_OUTPUT
+        echo "HOPSFS_VERSION=${HOPSFS_VERSION}" >> $GITHUB_OUTPUT
+
+    - name: Build hudi and hudi-packaging
+      shell: bash
+      if: steps.to_build_or_not_to_build.outputs.BUILD_HUDI == 'true'
+      working-directory: ${{ github.workspace }}/hudi
+      env:
+        POM_VERSION: ${{ env.POM_VERSION }}
+        M2_HOME: ~/.m2
+      run: |
+        mvn -s ~/.m2/settings.xml -Denforcer.skip=true -Drat.skip=true -Dcheckstyle.skip=true clean install deploy -Pspark3.5 -DskipTests 
+
+    - name: Upload hudi-packaging artifact to Nexus
+      shell: bash
+      if: steps.to_build_or_not_to_build.outputs.BUILD_HUDI == 'true'
+      working-directory: ${{ github.workspace }}/hudi
+      run: |
+        ls -lh
+        curl -u ${{ vars.NEXUS_HARBOR_USER }}:${{ secrets.NEXUS_HARBOR_PASSWORD }} --upload-file packaging/hudi-spark-bundle/target/hudi-spark3.5-bundle_2.12-${POM_VERSION}.jar "${HUDI_SPARK_JAR_URL}"
+        curl -u ${{ vars.NEXUS_HARBOR_USER }}:${{ secrets.NEXUS_HARBOR_PASSWORD }} --upload-file packaging/hudi-utilities-slim-bundle/target/hudi-utilities-slim-bundle_2.12-${POM_VERSION}.jar "${HUDI_UTILS_JAR_URL}"

docker/hoodie/hadoop/base/pom.xml

@@ -18,7 +18,7 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
-    <groupId>org.apache.hudi</groupId>
+    <groupId>io.hops.hudi</groupId>
     <version>1.0.2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
@@ -37,7 +37,7 @@
   <dependencyManagement>
     <dependencies>
       <dependency>
-        <groupId>org.apache.hudi</groupId>
+        <groupId>io.hops.hudi</groupId>
         <artifactId>hudi-hadoop-docker</artifactId>
         <version>${project.version}</version>
         <type>pom</type>

docker/hoodie/hadoop/base_java11/pom.xml

@@ -19,7 +19,7 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
-    <groupId>org.apache.hudi</groupId>
+    <groupId>io.hops.hudi</groupId>
     <version>1.0.2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
@@ -38,7 +38,7 @@
   <dependencyManagement>
     <dependencies>
       <dependency>
-        <groupId>org.apache.hudi</groupId>
+        <groupId>io.hops.hudi</groupId>
         <artifactId>hudi-hadoop-docker</artifactId>
         <version>${project.version}</version>
         <type>pom</type>

docker/hoodie/hadoop/datanode/pom.xml

@@ -18,7 +18,7 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
-    <groupId>org.apache.hudi</groupId>
+    <groupId>io.hops.hudi</groupId>
     <version>1.0.2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
@@ -36,7 +36,7 @@
   <dependencyManagement>
     <dependencies>
       <dependency>
-        <groupId>org.apache.hudi</groupId>
+        <groupId>io.hops.hudi</groupId>
         <artifactId>hudi-hadoop-base-docker</artifactId>
         <version>${project.version}</version>
         <type>pom</type>

docker/hoodie/hadoop/historyserver/pom.xml

@@ -18,7 +18,7 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
-    <groupId>org.apache.hudi</groupId>
+    <groupId>io.hops.hudi</groupId>
     <version>1.0.2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
@@ -36,7 +36,7 @@
   <dependencyManagement>
     <dependencies>
       <dependency>
-        <groupId>org.apache.hudi</groupId>
+        <groupId>io.hops.hudi</groupId>
         <artifactId>hudi-hadoop-base-docker</artifactId>
         <version>${project.version}</version>
         <type>pom</type>

docker/hoodie/hadoop/hive_base/pom.xml

@@ -18,7 +18,7 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
-    <groupId>org.apache.hudi</groupId>
+    <groupId>io.hops.hudi</groupId>
     <version>1.0.2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
@@ -36,11 +36,21 @@
   <dependencyManagement>
     <dependencies>
       <dependency>
-        <groupId>org.apache.hudi</groupId>
+        <groupId>io.hops.hudi</groupId>
         <artifactId>hudi-hadoop-base-docker</artifactId>
         <version>${project.version}</version>
         <type>pom</type>
         <scope>import</scope>
+        <exclusions>
+          <exclusion>
+            <groupId>io.hops.hive</groupId>
+            <artifactId>hive-llap-server</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>io.hops.hive</groupId>
+            <artifactId>hive-llap-tez</artifactId>
+          </exclusion>
+        </exclusions>
       </dependency>
     </dependencies>
   </dependencyManagement>

docker/hoodie/hadoop/namenode/pom.xml

@@ -18,7 +18,7 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
-    <groupId>org.apache.hudi</groupId>
+    <groupId>io.hops.hudi</groupId>
     <version>1.0.2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
@@ -36,7 +36,7 @@
   <dependencyManagement>
     <dependencies>
       <dependency>
-        <groupId>org.apache.hudi</groupId>
+        <groupId>io.hops.hudi</groupId>
         <artifactId>hudi-hadoop-base-docker</artifactId>
         <version>${project.version}</version>
         <type>pom</type>

docker/hoodie/hadoop/pom.xml

@@ -18,7 +18,7 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <parent>
     <artifactId>hudi</artifactId>
-    <groupId>org.apache.hudi</groupId>
+    <groupId>io.hops.hudi</groupId>
     <version>1.0.2</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
@@ -45,22 +45,22 @@
 
   <dependencies>
     <dependency>
-      <groupId>org.apache.hudi</groupId>
+      <groupId>io.hops.hudi</groupId>
       <artifactId>hudi-hadoop-mr-bundle</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>
-      <groupId>org.apache.hudi</groupId>
+      <groupId>io.hops.hudi</groupId>
       <artifactId>hudi-hive-sync-bundle</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>
-      <groupId>org.apache.hudi</groupId>
+      <groupId>io.hops.hudi</groupId>
       <artifactId>hudi-spark${sparkbundle.version}-bundle_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>
-      <groupId>org.apache.hudi</groupId>
+      <groupId>io.hops.hudi</groupId>
       <artifactId>hudi-utilities-bundle_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>

docker/hoodie/hadoop/prestobase/pom.xml

@@ -19,7 +19,7 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <parent>
     <artifactId>hudi-hadoop-docker</artifactId>
-    <groupId>org.apache.hudi</groupId>
+    <groupId>io.hops.hudi</groupId>
     <version>1.0.2</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
@@ -36,7 +36,7 @@
   <dependencyManagement>
     <dependencies>
       <dependency>
-        <groupId>org.apache.hudi</groupId>
+        <groupId>io.hops.hudi</groupId>
         <artifactId>hudi-hadoop-base-docker</artifactId>
         <version>${project.version}</version>
         <type>pom</type>