From 7488a5dc27d49c795372aaf419bb7a0f7b4be1e4 Mon Sep 17 00:00:00 2001
From: Ravi Dalal <12639199+ravi-dalal@users.noreply.github.com>
Date: Mon, 5 May 2025 12:05:30 -0700
Subject: [PATCH] upgraded spark on ray on vertex ai notebook to 2.42.0 version
 (#4027)

* upgraded spark on ray on vertex ai notebook to 2.42.0 version

* upgraded spark on ray on vertex ai notebook to 2.42.0 version

* upgraded spark on ray on vertex ai notebook to 2.42.0 version

* upgraded spark on ray on vertex ai notebook to 2.42.0 version
---
 .../spark_on_ray_on_vertex_ai.ipynb               | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/notebooks/official/ray_on_vertex_ai/spark_on_ray_on_vertex_ai.ipynb b/notebooks/official/ray_on_vertex_ai/spark_on_ray_on_vertex_ai.ipynb
index c11ec75dc..b0550ef7b 100644
--- a/notebooks/official/ray_on_vertex_ai/spark_on_ray_on_vertex_ai.ipynb
+++ b/notebooks/official/ray_on_vertex_ai/spark_on_ray_on_vertex_ai.ipynb
@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2024 Google LLC\n",
+        "# Copyright 2025 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -156,9 +156,9 @@
       },
       "outputs": [],
       "source": [
-        "! pip3 install --upgrade --quiet google-cloud-aiplatform[ray]==1.59.0\n",
+        "! pip3 install --upgrade --quiet google-cloud-aiplatform[ray]\n",
         "! pip3 install --upgrade --quiet pyspark\n",
-        "! pip3 install --upgrade --quiet ray[all]==2.9.3\n",
+        "! pip3 install --upgrade --quiet ray[all]==2.42.0\n",
         "! pip3 install --upgrade --quiet raydp"
       ]
     },
@@ -317,11 +317,11 @@
       "source": [
         "%%writefile {DOCKER_DIR}/Dockerfile\n",
         "\n",
-        "FROM us-docker.pkg.dev/vertex-ai/training/ray-cpu.2-9.py310:latest\n",
+        "FROM us-docker.pkg.dev/vertex-ai/training/ray-cpu.2-42.py310:latest\n",
         "\n",
         "RUN apt-get update -y \\\n",
-        "    && apt-get install openjdk-21-jdk -y \\\n",
-        "    && pip install --no-cache-dir raydp pyarrow==14.0 pandas==2.1.0"
+        "    && apt-get install openjdk-17-jdk -y \\\n",
+        "    && pip install --no-cache-dir raydp pyarrow pandas"
       ]
     },
     {
@@ -591,7 +591,8 @@
         "            time.sleep(30)\n",
         "            func()\n",
         "            break\n",
-        "        except Exception:\n",
+        "        except Exception as e:\n",
+        "            print(f\"An error occurred: {e}\")\n",
         "            continue\n",
         "\n",
         "\n",