feat: support for Spark 4 (#589)

* feat: support for Spark 4

* update ny-tlc-report

* chore: Generated commit to update templated files since the last template run up to stackabletech/operator-templating@e3ffe71 (#590)

Reference-to: stackabletech/operator-templating@e3ffe71 (chore: SDP 25.7 templating updates)

* chore: Bump Rust dependencies pre-25.7.0 (#592)

* feat(helm): Add RBAC rule for automatic cluster domain detection

* chore: Bump stackable-operator to 0.94.0 and update other dependencies

* chore: Update changelog

* chore: Add sparkhistory and shs shortnames

* fix the iceberg test

* update supported versions

* update docs, examples, getting started

* skip logging tests for spark 4

* Apply suggestions from code review

Co-authored-by: Andrew Kenworthy <[email protected]>

* update supported versions page

* update changelog

---------

Co-authored-by: Stacky McStackface <[email protected]>
Co-authored-by: Nick <[email protected]>
Co-authored-by: Andrew Kenworthy <[email protected]>
Co-authored-by: Andrew Kenworthy <[email protected]>

Author: 4 people

CHANGELOG.md

@@ -4,10 +4,15 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+### Added
+
+- Add experimental support for Spark 4 ([#589])
+
 ### Fixed
 
 - SparkConnectServer: The `imagePullSecret` is now correctly passed to Spark executor pods ([#603]).
 
+[#589]: https://github.com/stackabletech/spark-k8s-operator/pull/589
 [#603]: https://github.com/stackabletech/spark-k8s-operator/pull/603
 
 ## [25.7.0] - 2025-07-23

docs/modules/spark-k8s/examples/example-history-app.yaml

@@ -5,7 +5,7 @@ metadata:
   name: spark-pi-s3-1
 spec:
   sparkImage:
-    productVersion: 3.5.6
+    productVersion: 4.0.0
     pullPolicy: IfNotPresent
   mode: cluster
   mainClass: org.apache.spark.examples.SparkPi

docs/modules/spark-k8s/examples/example-history-server.yaml

@@ -5,7 +5,7 @@ metadata:
   name: spark-history
 spec:
   image:
-    productVersion: 3.5.6
+    productVersion: 4.0.0
   logFileDirectory:  # <1>
     s3:
       prefix: eventlogs/  # <2>

docs/modules/spark-k8s/examples/example-pvc.yaml

@@ -1,8 +1,8 @@
 ---
 apiVersion: v1
-kind: PersistentVolume
+kind: PersistentVolume  # <1>
 metadata:
-  name: pv-ksv # <1>
+  name: pv-ksv
 spec:
   storageClassName: standard
   accessModes:
@@ -13,31 +13,31 @@ spec:
     path: /some-host-location
 ---
 apiVersion: v1
-kind: PersistentVolumeClaim
+kind: PersistentVolumeClaim # <2>
 metadata:
-  name: pvc-ksv # <2>
+  name: pvc-ksv
 spec:
-  volumeName: pv-ksv # <1>
+  volumeName: pv-ksv # <3>
   accessModes:
     - ReadWriteOnce
   resources:
     requests:
       storage: 1Gi
 ---
 apiVersion: batch/v1
-kind: Job
+kind: Job # <4>
 metadata:
   name: aws-deps
 spec:
   template:
     spec:
       restartPolicy: Never
       volumes:
-        - name: job-deps # <3>
+        - name: job-deps
           persistentVolumeClaim:
-            claimName: pvc-ksv # <2>
+            claimName: pvc-ksv # <5>
       containers:
         - name: aws-deps
           volumeMounts:
-            - name: job-deps # <4>
+            - name: job-deps
               mountPath: /stackable/spark/dependencies

docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml

@@ -6,7 +6,7 @@ metadata:
   namespace: default
 spec:
   sparkImage:
-    productVersion: 3.5.6
+    productVersion: 4.0.0
   mode: cluster
   mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.1.0.jar # <3>
   mainClass: tech.stackable.demo.spark.NYTLCReport

docs/modules/spark-k8s/examples/example-sparkapp-image.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   image: oci.stackable.tech/stackable/ny-tlc-report:0.2.0 # <1>
   sparkImage:
-    productVersion: 3.5.6
+    productVersion: 4.0.0
   mode: cluster
   mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py # <2>
   args:

docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml

@@ -6,28 +6,30 @@ metadata:
   namespace: default
 spec:
   sparkImage:
-    productVersion: 3.5.6
+    productVersion: 4.0.0
   mode: cluster
-  mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.0-SNAPSHOT.jar # <1>
+  mainApplicationFile: s3a://my-bucket/app.jar # <1>
   mainClass: org.example.App # <2>
-  args:
-    - "'s3a://nyc-tlc/trip data/yellow_tripdata_2021-07.csv'"
   sparkConf: # <3>
-    "spark.hadoop.fs.s3a.aws.credentials.provider": "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"
     "spark.driver.extraClassPath": "/dependencies/jars/*"
     "spark.executor.extraClassPath": "/dependencies/jars/*"
   volumes:
     - name: job-deps # <4>
       persistentVolumeClaim:
         claimName: pvc-ksv
-  driver:
+  job:
     config:
       volumeMounts:
         - name: job-deps
           mountPath: /dependencies # <5>
+  driver:
+    config:
+      volumeMounts:
+        - name: job-deps
+          mountPath: /dependencies # <6>
   executor:
     replicas: 3
     config:
       volumeMounts:
         - name: job-deps
-          mountPath: /dependencies # <5>
+          mountPath: /dependencies # <7>

docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml

@@ -5,7 +5,7 @@ metadata:
   name: example-sparkapp-s3-private
 spec:
   sparkImage:
-    productVersion: 3.5.6
+    productVersion: 4.0.0
   mode: cluster
   mainApplicationFile: s3a://my-bucket/spark-examples.jar # <1>
   mainClass: org.apache.spark.examples.SparkPi # <2>

docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml

@@ -6,7 +6,7 @@ metadata:
   namespace: default
 spec:
   sparkImage:
-    productVersion: 3.5.6
+    productVersion: 4.0.0
   mode: cluster
   mainApplicationFile: local:///stackable/spark/examples/src/main/python/streaming/hdfs_wordcount.py
   args:

docs/modules/spark-k8s/examples/getting_started/getting_started.sh

@@ -56,7 +56,7 @@ metadata:
   namespace: default
 spec:
   sparkImage:
-    productVersion: 3.5.6
+    productVersion: 4.0.0
   mode: cluster
   mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py
   driver: