chore(deps): update dependency apache/spark to v4

Author: renovate[bot]

.github/workflows/main-workflow.yml

@@ -103,7 +103,7 @@ jobs:
       base_image: python-minimal
       python_version_1: 3.12.9
       python_version_2: 3.11.12
-      spark_version: 3.5.5
+      spark_version: 4.0.0
       build_gpu: false
     secrets: inherit
   jupyter-pyspark:
@@ -115,7 +115,7 @@ jobs:
       base_image: pyspark
       python_version_1: 3.12.9
       python_version_2: 3.11.12
-      spark_version: 3.5.5
+      spark_version: 4.0.0
       build_gpu: false
     secrets: inherit
   vscode-pyspark:
@@ -127,7 +127,7 @@ jobs:
       base_image: pyspark
       python_version_1: 3.12.9
       python_version_2: 3.11.12
-      spark_version: 3.5.5
+      spark_version: 4.0.0
       build_gpu: false
     secrets: inherit
   jupyter-pytorch:
@@ -189,7 +189,7 @@ jobs:
       base_image: r-minimal
       r_version_1: 4.5.0
       r_version_2: 4.4.3
-      spark_version: 3.5.5
+      spark_version: 4.0.0
       build_gpu: false
     secrets: inherit
   rstudio-sparkr:
@@ -201,7 +201,7 @@ jobs:
       base_image: sparkr
       r_version_1: 4.5.0
       r_version_2: 4.4.3
-      spark_version: 3.5.5
+      spark_version: 4.0.0
       build_gpu: false
     secrets: inherit
   r-python-julia:

spark/Dockerfile

@@ -8,7 +8,7 @@ ARG BASE_IMAGE
 ENV JAVA_VERSION="17"
 ENV JAVA_HOME="/usr/lib/jvm/java-${JAVA_VERSION}-openjdk-amd64"
 
-ARG SPARK_VERSION="3.5.5"
+ARG SPARK_VERSION="4.0.0"
 ENV SPARK_VERSION=${SPARK_VERSION}
 
 ENV HADOOP_HOME="/usr/local/lib/hadoop"