Update Dockerfile

Dockerfile

@@ -5,6 +5,7 @@ FROM ubuntu:18.04
 ENV NB_USER jovyan
 ENV NB_UID 1000
 ENV HOME /home/${NB_USER}
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
 
 # Install required packages
 RUN apt-get update && apt-get install -y \
@@ -28,20 +29,17 @@ RUN apt-get update && apt-get install -y \
     ant \
     ca-certificates-java \
     && apt-get clean \
-    && update-ca-certificates -f;
+    && update-ca-certificates -f
 
 # Install Python 3.8 and pip
 RUN add-apt-repository ppa:deadsnakes/ppa \
     && apt-get update \
     && apt-get install -y python3.8 python3-pip \
-    && apt-get clean;
+    && apt-get clean
 
 # Set up JAVA_HOME
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
-RUN mkdir -p ${HOME} \
-    && echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> ${HOME}/.bashrc \
-    && chown -R ${NB_UID}:${NB_UID} ${HOME}
-
+RUN echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> /etc/profile \
+    && echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> /etc/profile
 # Create a new user named "jovyan" with user ID 1000
 RUN useradd -m -u ${NB_UID} ${NB_USER}
 
@@ -52,6 +50,10 @@ USER ${NB_USER}
 ENV HOME=/home/${NB_USER} \
     PATH=/home/${NB_USER}/.local/bin:$PATH
 
+# Set up PySpark to use Python 3.8 for both driver and workers
+ENV PYSPARK_PYTHON=/usr/bin/python3.8
+ENV PYSPARK_DRIVER_PYTHON=/usr/bin/python3.8
+
 # Set the working directory to the user's home directory
 WORKDIR ${HOME}