debug: timeout fix

notebooks/community/sdk/sdk_pytorch_torchrun_custom_container_training_imagenet.ipynb

@@ -567,7 +567,7 @@
       "outputs": [],
       "source": [
         "%%writefile trainer/Dockerfile\n",
-        "FROM gcr.io/deeplearning-platform-release/pytorch-gpu.1-13:m102\n",
+        "FROM gcr.io/deeplearning-platform-release/pytorch-gpu.1-12\n",
         "\n",
         "RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \\\n",
         "    # Install reduction server plugin on GPU containers. google-fast-socket is\n",
@@ -588,31 +588,16 @@
         "RUN apt-get update -y && \\\n",
         "    apt-get install -y curl gnupg telnet nano net-tools iputils-ping\n",
         "\n",
-        "# Set ETCD version\n",
-        "ARG ETCD_VER=v2.3.0\n",
-        "# Choose either URL\n",
-        "ARG GOOGLE_URL=https://storage.googleapis.com/etcd\n",
-        "ARG GITHUB_URL=https://github.com/etcd-io/etcd/releases/download\n",
-        "# Set ETCD URL to download from\n",
-        "ARG DOWNLOAD_URL=$GOOGLE_URL\n",
         "\n",
-        "# Install ETCD\n",
-        "RUN mkdir -p /tmp/etcd-download-test && \\\n",
-        "    curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz && \\\n",
-        "    tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1 && \\\n",
-        "    rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz\n",
-        "\n",
-        "# Copy training application code\n",
         "COPY . /trainer\n",
         "\n",
         "WORKDIR /trainer\n",
         "\n",
-        "# Install dependencies\n",
         "RUN pip install -r requirements.txt\n",
         "\n",
         "RUN chmod 777 main.sh\n",
         "\n",
-        "# Download data to the container\n",
+        "# download data to the container\n",
         "RUN wget -q -P /trainer/data https://image-net.org/data/tiny-imagenet-200.zip\n",
         "RUN unzip -q /trainer/data/tiny-imagenet-200.zip\n",
         "RUN rm /trainer/data/tiny-imagenet-200.zip\n",
@@ -629,8 +614,8 @@
       "outputs": [],
       "source": [
         "%%writefile trainer/requirements.txt\n",
-        "torch==1.13.0\n",
-        "torchvision==0.14.0\n",
+        "torch==1.12.0\n",
+        "torchvision==0.13.0\n",
         "tensorboard==2.5.0\n",
         "protobuf==3.20.*\n",
         "python-etcd\n",
@@ -690,17 +675,30 @@
         "setup_etcd() {\n",
         "    HOST_IP=$1\n",
         "    # Start a local instane of ETCD v2 \n",
+        "    ETCD_VER=v2.3.0 #v3.5.6\n",
         "    export ETCD_ENABLE_V2=true\n",
         "    export ETCDCTL_API=2\n",
         "\n",
+        "    # choose either URL\n",
+        "    GOOGLE_URL=https://storage.googleapis.com/etcd\n",
+        "    GITHUB_URL=https://github.com/etcd-io/etcd/releases/download\n",
+        "    DOWNLOAD_URL=${GOOGLE_URL}\n",
+        "\n",
+        "    rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz\n",
+        "    rm -rf /tmp/etcd-download-test && mkdir -p /tmp/etcd-download-test\n",
+        "\n",
+        "    curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz\n",
+        "    tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1\n",
+        "    rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz\n",
+        "\n",
         "    /tmp/etcd-download-test/etcd --name s1 --data-dir /tmp/etcd-download-test/s1  \\\n",
         "    --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://$HOST_IP:2379 \\\n",
         "    --listen-peer-urls http://0.0.0.0:2380 --initial-advertise-peer-urls http://$HOST_IP:2380 \\\n",
         "    --initial-cluster s1=http://$HOST_IP:2380 --initial-cluster-token tkn \\\n",
         "    --initial-cluster-state new &> /tmp/etcd-download-test/node.log &\n",
         "\n",
-        "    /tmp/etcd-download-test/etcd --version\n",
-        "    /tmp/etcd-download-test/etcdctl --version\n",
+        "    sudo /tmp/etcd-download-test/etcd --version\n",
+        "    sudo /tmp/etcd-download-test/etcdctl --version\n",
         "}\n",
         "\n",
         "\n",

notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb

@@ -29,7 +29,7 @@
         "id": "l2mMvIUG9meX"
       },
       "source": [
-        "# Profile model training performance using Vertex AI TensorBoard Profiler\n",
+        "# Profile model training performance using Profiler\n",
         "\n",
         "<table align=\"left\">\n",
         "\n",
@@ -49,7 +49,7 @@
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
-        "  </td>\n",
+        "  </td>                                                                                               \n",
         "</table>"
       ]
     },
@@ -331,16 +331,22 @@
         "! gcloud projects add-iam-policy-binding $PROJECT_ID \\\n",
         "        --member=\"serviceAccount:$SERVICE_ACCOUNT\" \\\n",
         "        --role=\"roles/storage.admin\" \\\n",
-        "        --quiet\n",
-        "\n",
+        "        --quiet"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "S_8_5jm-Gk6w"
+      },
+      "outputs": [],
+      "source": [
         "# Grant AI Platform permission.\n",
         "! gcloud projects add-iam-policy-binding $PROJECT_ID \\\n",
         "        --member=\"serviceAccount:$SERVICE_ACCOUNT\" \\\n",
         "        --role=\"roles/aiplatform.user\" \\\n",
-        "        --quiet\n",
-        "\n",
-        "! gcloud projects get-iam-policy $PROJECT_ID \\\n",
-        "        --filter=bindings.members:serviceAccount:$SERVICE_ACCOUNT"
+        "        --quiet"
       ]
     },
     {
@@ -762,7 +768,7 @@
         "WORKDIR /root\n",
         "\n",
         "# Installs additional packages as you need.\n",
-        "RUN pip3 install google-cloud-aiplatform[cloud_profiler]>=1.19.1\n",
+        "RUN pip3 install google-cloud-aiplatform[cloud_profiler]\n",
         "\n",
         "# Copies the trainer code to the docker image.\n",
         "RUN mkdir /root/trainer\n",
@@ -792,7 +798,7 @@
         "IMAGE_NAME = \"tensorboard-custom-container\"\n",
         "IMAGE_URI = f\"{REGION}-docker.pkg.dev/{PROJECT_ID}/{DOCKER_REPOSITORY}/{IMAGE_NAME}\"\n",
         "\n",
-        "! gcloud builds submit --project {PROJECT_ID} --region={REGION} --tag {IMAGE_URI} --timeout=3600s --quiet"
+        "! gcloud builds submit --project {PROJECT_ID} --region={REGION} --tag {IMAGE_URI} --timeout=60m --quiet"
       ]
     },
     {

notebooks/official/test.ipynb

@@ -0,0 +1,61 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "346219a0-c421-4986-89c0-1cab5f9896bc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import time"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fdeaadd2-900d-4580-a637-b0ec3c892ce7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "time.sleep(5 * 60)\n",
+    "print(\"5 mins\")\n",
+    "\n",
+    "time.sleep(5 * 60)\n",
+    "print(\"10 mins\")\n",
+    "\n",
+    "time.sleep(5 * 60)\n",
+    "print(\"15 mins\")\n",
+    "\n",
+    "time.sleep(5 * 60)\n",
+    "print(\"20 mins\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "environment": {
+   "kernel": "python3",
+   "name": "common-cpu.m95",
+   "type": "gcloud",
+   "uri": "gcr.io/deeplearning-platform-release/base-cpu:m95"
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}