Upgrade Spark to latest version, 3.1.2

- Moved to Python 3.9.2
- Kept Java 11, upgraded base docker image
- Direnv to use custom "use java" (via jabba)

Author: luisbelloch

.envrc

@@ -1,6 +1,6 @@
-layout pyenv 3.9.0
-use sdk java 11.0.9.hs-adpt
-export VBOXMANAGE=/Applications/VirtualBox.app/Contents/MacOS
-export SPARK_HOME=$(pwd)/.spark
-export PATH=$SPARK_HOME/bin:$VBOXMANAGE:$PATH
+layout pyenv 3.9.2
+use java [email protected]
+export SPARK_HOME="$(pwd)/.spark"
+PATH_add "$SPARK_HOME/bin"
 export PYSPARK_PYTHON=python3
+

.python-version

@@ -1 +1 @@
-3.9.0
+3.9.2

infra/docker/Dockerfile

@@ -6,9 +6,9 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends python3-software-properties python3-numpy curl && \
     rm -rf /var/lib/apt/lists/*
 
-ARG SPARK_VERSION=3.0.1
+ARG SPARK_VERSION=3.1.2
 ENV SPARK_HOME=/opt/spark
-RUN mkdir -p /opt/spark && curl -s https://downloads.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz | tar -xz -C "${SPARK_HOME}" --strip-components=1
+RUN mkdir -p /opt/spark && curl -s https://downloads.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.2.tgz | tar -xz -C "${SPARK_HOME}" --strip-components=1
 ENV PATH=$SPARK_HOME/bin:$SPARK_HOME/sbin:$PATH
 
 RUN cp "${SPARK_HOME}/conf/log4j.properties.template" "${SPARK_HOME}/conf/log4j.properties" && \

infra/docker/Makefile

@@ -1,5 +1,5 @@
-SPARK_VERSION:=3.0.1
-COURSE_VERSION:=2020.12
+SPARK_VERSION:=3.1.2
+COURSE_VERSION:=2021.10
 IMAGE_NAME:=luisbelloch/spark
 
 .PHONY: help

local_setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 set -euo pipefail
-SPARK_URL=${SPARK_URL:-https://downloads.apache.org/spark/spark-3.0.1/spark-3.0.1-bin-hadoop2.7.tgz}
+SPARK_URL=${SPARK_URL:-https://downloads.apache.org/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz}
 SPARK_PKG=${SPARK_URL##*/}
 SPARK_HOME=${SPARK_HOME:-$(pwd)/.spark}
 

playbook.yml

@@ -2,8 +2,8 @@
 - hosts: all
   vars:
     spark_home: /opt/spark
-    spark_pkg_name: spark-3.0.1-bin-hadoop2.7
-    spark_pkg_url: https://downloads.apache.org/spark/spark-3.0.1/spark-3.0.1-bin-hadoop2.7.tgz
+    spark_pkg_name: spark-3.1.2-bin-hadoop3.2
+    spark_pkg_url: https://downloads.apache.org/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz
 
   tasks:
     - name: Update all packages to the latest version