apache · yoda-mon · Nov 24, 2022
diff --git a/bigtop-deploy/puppet/manifests/jdk.pp b/bigtop-deploy/puppet/manifests/jdk.pp
@@ -57,3 +57,73 @@
     }
   }
 }
+
+define jdk::add($version) {
+  case $::operatingsystem {
+    /Debian/: {
+      include apt
+
+      package { "openjdk-${version}-jdk" :
+        ensure  => present,
+      }
+    }
+    /Ubuntu/: {
+      include apt
+
+      package { "openjdk-${version}-jdk" :
+        ensure  => present,
+      }
+    }
+    /(CentOS|Fedora|RedHat)/: {
+      package { "java-${version}-openjdk-devel" :
+        ensure => present
+      }
+    }
+  }
+}
+
+define jdk::use($version) {
+  case $::operatingsystem {
+    /Debian/: {
+      if versioncmp($version, "9") < 0 {
+        exec { "use java ${version}":
+          command => "update-java-alternatives --set adoptopenjdk-${version}-$(dpkg --print-architecture)",
+          path    => ['/usr/sbin', '/usr/bin', '/bin'],
+        }
+      } else {
+        exec { "use java ${version}":
+          command => "update-java-alternatives --set java-1.${version}.0-openjdk-$(dpkg --print-architecture)",
+          path    => ['/usr/sbin', '/usr/bin', '/bin'],
+        }
+      }
+    }
+    /Ubuntu/: {
+      exec { "use java ${version}":
+        command => "update-java-alternatives --set java-1.${version}.0-openjdk-$(dpkg --print-architecture)",
+        path    => ['/usr/sbin', '/usr/bin', '/bin'],
+      }
+    }
+    /(CentOS|Fedora|RedHat)/: {
+      if versioncmp($version, "9") < 0 {
+        $version_id = "1.$version.0"
+      } else {
+        $version_id = "$version"
+      }
+      exec { "use java ${version}":
+        command => "update-alternatives --set java java-${version_id}-openjdk.$(uname -m) \
+                    && update-alternatives --set javac java-${version_id}-openjdk.$(uname -m)",
+        path    => ['/usr/sbin', '/usr/bin', '/bin'],
+      }
+    }
+  }
+}
+
+define jdk::set_profile($component, $version) {
+  $upcased_component = upcase($component)
+  file { "/etc/profile.d/bigtop_${component}_javaversion.sh":
+    content => "export BIGTOP_${upcased_component}_JAVA_VERSION=${version}\n",
+    owner  => root,
+    group  => root,
+    mode   => "644",
+  }
+}
diff --git a/bigtop-deploy/puppet/modules/kafka/manifests/init.pp b/bigtop-deploy/puppet/modules/kafka/manifests/init.pp
@@ -15,7 +15,26 @@
 
 class kafka {
 
+  class check_java_version() {
+    $version = hiera('bigtop::kafka_java_version')
+    if versioncmp($version, '8') > 0 {
+      jdk::add{"java $version for kafka":
+        version => $version,
+        before => Jdk::Use["java $version for kafka"],
+      }
+      jdk::use{"java $version for kafka":
+        version => $version,
+        before => Jdk::Set_Profile["BIGTOP_KAFKA_JAVA_VERSION"]
+      }
+      jdk::set_profile{"BIGTOP_KAFKA_JAVA_VERSION":
+        component => 'kafka', version => $version,
+        before => Service["kafka-server"],
+      }
+    }
+  }
+
   class deploy ($roles) {
+    require kafka::check_java_version
     if ('kafka-server' in $roles) {
       include kafka::server
     }

diff --git a/bigtop-packages/src/common/bigtop-utils/bigtop-detect-component-specific-javahome b/bigtop-packages/src/common/bigtop-utils/bigtop-detect-component-specific-javahome
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# If BIGTOP_{COMPONENT}_JAVA_VERSION is set,
+# this script try to overide JAVA_HOME
+
+COMPONENT=$1
+ENV_VAL_NAME="BIGTOP_${COMPONENT^^}_JAVA_VERSION"
+
+
+if [ -f "/etc/profile.d/bigtop_${COMPONENT}_javaversion.sh" ]; then
+  . /etc/profile.d/bigtop_${COMPONENT}_javaversion.sh
+fi
+
+BIGTOP_COMPONENT_JAVA_VERSION=$(eval "echo \${$ENV_VAL_NAME}")
+logger "java version for ${COMPONENT} is ${BIGTOP_COMPONENT_JAVA_VERSION}"
+
+if [ -n "$BIGTOP_COMPONENT_JAVA_VERSION" ]; then
+  JAVA_HOME_CANDIDATES=(
+    "/usr/lib/jvm/java-1.${BIGTOP_COMPONENT_JAVA_VERSION}.0-openjdk"
+    "/usr/lib/jvm/java-${BIGTOP_COMPONENT_JAVA_VERSION}-openjdk"
+    "/usr/lib/jvm/adoptopenjdk-${BIGTOP_COMPONENT_JAVA_VERSION}-hotspot"
+  )  # considering only OpenJDK or AdoptOpenJDK
+  for candidate_regex in ${JAVA_HOME_CANDIDATES[@]} ; do
+    for candidate in `ls -rvd ${candidate_regex}* 2>/dev/null`; do
+      if [ -e ${candidate}/bin/java ]; then
+        export JAVA_HOME=${candidate}
+        break 2
+      fi
+    done
+  done
+fi
diff --git a/bigtop-packages/src/common/kafka/install_kafka.sh b/bigtop-packages/src/common/kafka/install_kafka.sh
@@ -174,6 +174,8 @@ do
 
 # Autodetect JAVA_HOME if not defined
 . /usr/lib/bigtop-utils/bigtop-detect-javahome
+# Overwrite JAVA_HOME if BIGTOP_KAFKA_JAVA_VERSION is set
+. /usr/lib/bigtop-utils/bigtop-detect-component-specific-javahome kafka
 
 exec $LIB_DIR/bin/$file "\$@"
 EOF

diff --git a/bigtop-packages/src/common/kafka/kafka-server.svc b/bigtop-packages/src/common/kafka/kafka-server.svc
@@ -28,6 +28,8 @@ generate_start() {
 
 cat <<'__EOT__'
 start() {
+    # Overwrite JAVA_HOME if BIGTOP_KAFKA_JAVA_VERSION is set
+    . /usr/lib/bigtop-utils/bigtop-detect-component-specific-javahome kafka
     [ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED
     log_success_msg "Starting $DESC (${DAEMON}): "
 

diff --git a/bigtop-packages/src/deb/bigtop-utils/rules b/bigtop-packages/src/deb/bigtop-utils/rules
@@ -36,5 +36,6 @@ override_dh_auto_install:
 	install -p -m 755 debian/bigtop-detect-javalibs debian/bigtop-utils/usr/lib/bigtop-utils/
 	install -p -m 755 debian/bigtop-detect-classpath debian/bigtop-utils/usr/lib/bigtop-utils/
 	install -p -m 755 debian/bigtop-monitor-service debian/bigtop-utils/usr/lib/bigtop-utils/
+	install -p -m 755 debian/bigtop-detect-component-specific-javahome debian/bigtop-utils/usr/lib/bigtop-utils/
 	install -d -p -m 755 debian/bigtop-utils/etc/default
 	install -p -m 644 debian/bigtop-utils.default debian/bigtop-utils/etc/default/bigtop-utils
diff --git a/bigtop-packages/src/rpm/bigtop-utils/SPECS/bigtop-utils.spec b/bigtop-packages/src/rpm/bigtop-utils/SPECS/bigtop-utils.spec
@@ -32,6 +32,7 @@ Source2:    bigtop-utils.default
 Source3:    bigtop-detect-javalibs
 Source4:    bigtop-detect-classpath
 Source5:    bigtop-monitor-service
+Source6:    bigtop-detect-component-specific-javahome
 Requires:   bash
 
 # "which" command is needed for a lot of projects.
@@ -54,6 +55,7 @@ install -p -m 644 %{SOURCE2} .
 install -p -m 644 %{SOURCE3} .
 install -p -m 644 %{SOURCE4} .
 install -p -m 644 %{SOURCE5} .
+install -p -m 644 %{SOURCE6} .
 
 %build
 
@@ -66,6 +68,7 @@ install -p -m 755 %{SOURCE0} $RPM_BUILD_ROOT%{lib_dir}/
 install -p -m 755 %{SOURCE3} $RPM_BUILD_ROOT%{lib_dir}/
 install -p -m 755 %{SOURCE4} $RPM_BUILD_ROOT%{lib_dir}/
 install -p -m 755 %{SOURCE5} $RPM_BUILD_ROOT%{lib_dir}/
+install -p -m 755 %{SOURCE6} $RPM_BUILD_ROOT%{lib_dir}/
 install -p -m 644 %{SOURCE2} $RPM_BUILD_ROOT/etc/default/bigtop-utils
 
 %clean

diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
@@ -41,6 +41,7 @@ usage() {
     echo "                                                   NEXUS_URL is optional. If not specified, default to http://NEXUS_IP:8081/nexus"
     echo "                                                   Where NEXUS_IP is the ip of container named nexus"
     echo "       -p, --provision                           - Deploy configuration changes"
+    echo "       --[component]-java-version JAVA_VERSION   - Configure preferable Java version that used by supported components"
     echo "       -r, --repo REPO_URL                       - Overwrite the yum/apt repo defined in config file"
     echo "       -s, --smoke-tests COMPONENTS              - Run Bigtop smoke tests"
     echo "                                                   COMPONENTS is optional. If not specified, default to smoke_test_components in config file"
@@ -159,6 +160,7 @@ bigtop::bigtop_repo_gpg_check: $gpg_check
 hadoop_cluster_node::cluster_components: $3
 hadoop_cluster_node::cluster_nodes: [$node_list]
 hadoop::common_yarn::yarn_resourcemanager_scheduler_class: org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler
+$ADDITIONAL_CONFIG
 EOF
 }
 
@@ -329,6 +331,8 @@ if [ -e .provision_id ]; then
     PROVISION_ID=`cat .provision_id`
 fi
 
+ADDITIONAL_CONFIG=""  # placeholder to pass additional config to generate_config()
+
 while [ $# -gt 0 ]; do
     case "$1" in
     -c|--create)
@@ -430,6 +434,16 @@ while [ $# -gt 0 ]; do
     -p|--provision)
         provision
         shift;;
+    --*-java-version)
+        if [ $# -lt 2 ]; then
+          log "No java version specified"
+          usage
+        fi
+        java_version_component=$(echo "$1" | cut -d- -f3)
+        java_version_version="$2"
+        java_version_config="bigtop::${java_version_component}_java_version: \"${java_version_version}\"\n"
+        ADDITIONAL_CONFIG="${ADDITIONAL_CONFIG}${java_version_config}"
+        shift 2;;
     -r|--repo)
         if [ $# -lt 2 ]; then
           log "No yum/apt repo specified"
@@ -455,6 +469,8 @@ while [ $# -gt 0 ]; do
     esac
 done
 
+ADDITIONAL_CONFIG=$(echo -e $ADDITIONAL_CONFIG)  # process newline
+
 if [ "$READY_TO_LAUNCH" = true ]; then
     create $NUM_INSTANCES
 fi