diff --git a/bigtop-deploy/puppet/manifests/jdk.pp b/bigtop-deploy/puppet/manifests/jdk.pp index 895a81f502..29df7c04ee 100644 --- a/bigtop-deploy/puppet/manifests/jdk.pp +++ b/bigtop-deploy/puppet/manifests/jdk.pp @@ -57,3 +57,73 @@ } } } + +define jdk::add($version) { + case $::operatingsystem { + /Debian/: { + include apt + + package { "openjdk-${version}-jdk" : + ensure => present, + } + } + /Ubuntu/: { + include apt + + package { "openjdk-${version}-jdk" : + ensure => present, + } + } + /(CentOS|Fedora|RedHat)/: { + package { "java-${version}-openjdk-devel" : + ensure => present + } + } + } +} + +define jdk::use($version) { + case $::operatingsystem { + /Debian/: { + if versioncmp($version, "9") < 0 { + exec { "use java ${version}": + command => "update-java-alternatives --set adoptopenjdk-${version}-$(dpkg --print-architecture)", + path => ['/usr/sbin', '/usr/bin', '/bin'], + } + } else { + exec { "use java ${version}": + command => "update-java-alternatives --set java-1.${version}.0-openjdk-$(dpkg --print-architecture)", + path => ['/usr/sbin', '/usr/bin', '/bin'], + } + } + } + /Ubuntu/: { + exec { "use java ${version}": + command => "update-java-alternatives --set java-1.${version}.0-openjdk-$(dpkg --print-architecture)", + path => ['/usr/sbin', '/usr/bin', '/bin'], + } + } + /(CentOS|Fedora|RedHat)/: { + if versioncmp($version, "9") < 0 { + $version_id = "1.$version.0" + } else { + $version_id = "$version" + } + exec { "use java ${version}": + command => "update-alternatives --set java java-${version_id}-openjdk.$(uname -m) \ + && update-alternatives --set javac java-${version_id}-openjdk.$(uname -m)", + path => ['/usr/sbin', '/usr/bin', '/bin'], + } + } + } +} + +define jdk::set_profile($component, $version) { + $upcased_component = upcase($component) + file { "/etc/profile.d/bigtop_${component}_javaversion.sh": + content => "export BIGTOP_${upcased_component}_JAVA_VERSION=${version}\n", + owner => root, + group => root, + mode => "644", + } +} \ No newline at end of file diff --git a/bigtop-deploy/puppet/modules/kafka/manifests/init.pp b/bigtop-deploy/puppet/modules/kafka/manifests/init.pp index f13dec1267..fec783193a 100644 --- a/bigtop-deploy/puppet/modules/kafka/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/kafka/manifests/init.pp @@ -15,7 +15,26 @@ class kafka { + class check_java_version() { + $version = hiera('bigtop::kafka_java_version') + if versioncmp($version, '8') > 0 { + jdk::add{"java $version for kafka": + version => $version, + before => Jdk::Use["java $version for kafka"], + } + jdk::use{"java $version for kafka": + version => $version, + before => Jdk::Set_Profile["BIGTOP_KAFKA_JAVA_VERSION"] + } + jdk::set_profile{"BIGTOP_KAFKA_JAVA_VERSION": + component => 'kafka', version => $version, + before => Service["kafka-server"], + } + } + } + class deploy ($roles) { + require kafka::check_java_version if ('kafka-server' in $roles) { include kafka::server } diff --git a/bigtop-packages/src/common/bigtop-utils/bigtop-detect-component-specific-javahome b/bigtop-packages/src/common/bigtop-utils/bigtop-detect-component-specific-javahome new file mode 100644 index 0000000000..0f32021a09 --- /dev/null +++ b/bigtop-packages/src/common/bigtop-utils/bigtop-detect-component-specific-javahome @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# If BIGTOP_{COMPONENT}_JAVA_VERSION is set, +# this script try to overide JAVA_HOME + +COMPONENT=$1 +ENV_VAL_NAME="BIGTOP_${COMPONENT^^}_JAVA_VERSION" + + +if [ -f "/etc/profile.d/bigtop_${COMPONENT}_javaversion.sh" ]; then + . /etc/profile.d/bigtop_${COMPONENT}_javaversion.sh +fi + +BIGTOP_COMPONENT_JAVA_VERSION=$(eval "echo \${$ENV_VAL_NAME}") +logger "java version for ${COMPONENT} is ${BIGTOP_COMPONENT_JAVA_VERSION}" + +if [ -n "$BIGTOP_COMPONENT_JAVA_VERSION" ]; then + JAVA_HOME_CANDIDATES=( + "/usr/lib/jvm/java-1.${BIGTOP_COMPONENT_JAVA_VERSION}.0-openjdk" + "/usr/lib/jvm/java-${BIGTOP_COMPONENT_JAVA_VERSION}-openjdk" + "/usr/lib/jvm/adoptopenjdk-${BIGTOP_COMPONENT_JAVA_VERSION}-hotspot" + ) # considering only OpenJDK or AdoptOpenJDK + for candidate_regex in ${JAVA_HOME_CANDIDATES[@]} ; do + for candidate in `ls -rvd ${candidate_regex}* 2>/dev/null`; do + if [ -e ${candidate}/bin/java ]; then + export JAVA_HOME=${candidate} + break 2 + fi + done + done +fi diff --git a/bigtop-packages/src/common/kafka/install_kafka.sh b/bigtop-packages/src/common/kafka/install_kafka.sh index 94f83404b8..3a19aebf64 100644 --- a/bigtop-packages/src/common/kafka/install_kafka.sh +++ b/bigtop-packages/src/common/kafka/install_kafka.sh @@ -174,6 +174,8 @@ do # Autodetect JAVA_HOME if not defined . /usr/lib/bigtop-utils/bigtop-detect-javahome +# Overwrite JAVA_HOME if BIGTOP_KAFKA_JAVA_VERSION is set +. /usr/lib/bigtop-utils/bigtop-detect-component-specific-javahome kafka exec $LIB_DIR/bin/$file "\$@" EOF diff --git a/bigtop-packages/src/common/kafka/kafka-server.svc b/bigtop-packages/src/common/kafka/kafka-server.svc index 44b77d25f5..2e78eff6cc 100644 --- a/bigtop-packages/src/common/kafka/kafka-server.svc +++ b/bigtop-packages/src/common/kafka/kafka-server.svc @@ -28,6 +28,8 @@ generate_start() { cat <<'__EOT__' start() { + # Overwrite JAVA_HOME if BIGTOP_KAFKA_JAVA_VERSION is set + . /usr/lib/bigtop-utils/bigtop-detect-component-specific-javahome kafka [ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED log_success_msg "Starting $DESC (${DAEMON}): " diff --git a/bigtop-packages/src/deb/bigtop-utils/rules b/bigtop-packages/src/deb/bigtop-utils/rules index e55e985fa9..dc5777d46b 100644 --- a/bigtop-packages/src/deb/bigtop-utils/rules +++ b/bigtop-packages/src/deb/bigtop-utils/rules @@ -36,5 +36,6 @@ override_dh_auto_install: install -p -m 755 debian/bigtop-detect-javalibs debian/bigtop-utils/usr/lib/bigtop-utils/ install -p -m 755 debian/bigtop-detect-classpath debian/bigtop-utils/usr/lib/bigtop-utils/ install -p -m 755 debian/bigtop-monitor-service debian/bigtop-utils/usr/lib/bigtop-utils/ + install -p -m 755 debian/bigtop-detect-component-specific-javahome debian/bigtop-utils/usr/lib/bigtop-utils/ install -d -p -m 755 debian/bigtop-utils/etc/default install -p -m 644 debian/bigtop-utils.default debian/bigtop-utils/etc/default/bigtop-utils diff --git a/bigtop-packages/src/rpm/bigtop-utils/SPECS/bigtop-utils.spec b/bigtop-packages/src/rpm/bigtop-utils/SPECS/bigtop-utils.spec index 9a89091f86..938a7fd29d 100644 --- a/bigtop-packages/src/rpm/bigtop-utils/SPECS/bigtop-utils.spec +++ b/bigtop-packages/src/rpm/bigtop-utils/SPECS/bigtop-utils.spec @@ -32,6 +32,7 @@ Source2: bigtop-utils.default Source3: bigtop-detect-javalibs Source4: bigtop-detect-classpath Source5: bigtop-monitor-service +Source6: bigtop-detect-component-specific-javahome Requires: bash # "which" command is needed for a lot of projects. @@ -54,6 +55,7 @@ install -p -m 644 %{SOURCE2} . install -p -m 644 %{SOURCE3} . install -p -m 644 %{SOURCE4} . install -p -m 644 %{SOURCE5} . +install -p -m 644 %{SOURCE6} . %build @@ -66,6 +68,7 @@ install -p -m 755 %{SOURCE0} $RPM_BUILD_ROOT%{lib_dir}/ install -p -m 755 %{SOURCE3} $RPM_BUILD_ROOT%{lib_dir}/ install -p -m 755 %{SOURCE4} $RPM_BUILD_ROOT%{lib_dir}/ install -p -m 755 %{SOURCE5} $RPM_BUILD_ROOT%{lib_dir}/ +install -p -m 755 %{SOURCE6} $RPM_BUILD_ROOT%{lib_dir}/ install -p -m 644 %{SOURCE2} $RPM_BUILD_ROOT/etc/default/bigtop-utils %clean diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh index 0ba0133e36..3600f99af7 100755 --- a/provisioner/docker/docker-hadoop.sh +++ b/provisioner/docker/docker-hadoop.sh @@ -41,6 +41,7 @@ usage() { echo " NEXUS_URL is optional. If not specified, default to http://NEXUS_IP:8081/nexus" echo " Where NEXUS_IP is the ip of container named nexus" echo " -p, --provision - Deploy configuration changes" + echo " --[component]-java-version JAVA_VERSION - Configure preferable Java version that used by supported components" echo " -r, --repo REPO_URL - Overwrite the yum/apt repo defined in config file" echo " -s, --smoke-tests COMPONENTS - Run Bigtop smoke tests" echo " COMPONENTS is optional. If not specified, default to smoke_test_components in config file" @@ -159,6 +160,7 @@ bigtop::bigtop_repo_gpg_check: $gpg_check hadoop_cluster_node::cluster_components: $3 hadoop_cluster_node::cluster_nodes: [$node_list] hadoop::common_yarn::yarn_resourcemanager_scheduler_class: org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler +$ADDITIONAL_CONFIG EOF } @@ -329,6 +331,8 @@ if [ -e .provision_id ]; then PROVISION_ID=`cat .provision_id` fi +ADDITIONAL_CONFIG="" # placeholder to pass additional config to generate_config() + while [ $# -gt 0 ]; do case "$1" in -c|--create) @@ -430,6 +434,16 @@ while [ $# -gt 0 ]; do -p|--provision) provision shift;; + --*-java-version) + if [ $# -lt 2 ]; then + log "No java version specified" + usage + fi + java_version_component=$(echo "$1" | cut -d- -f3) + java_version_version="$2" + java_version_config="bigtop::${java_version_component}_java_version: \"${java_version_version}\"\n" + ADDITIONAL_CONFIG="${ADDITIONAL_CONFIG}${java_version_config}" + shift 2;; -r|--repo) if [ $# -lt 2 ]; then log "No yum/apt repo specified" @@ -455,6 +469,8 @@ while [ $# -gt 0 ]; do esac done +ADDITIONAL_CONFIG=$(echo -e $ADDITIONAL_CONFIG) # process newline + if [ "$READY_TO_LAUNCH" = true ]; then create $NUM_INSTANCES fi