debug: force use of newest cloud-build

* updating custom container with PyTorch v1.13

* moved etcd install to custom container build

.cloud-build/execute_changed_notebooks_helper.py

@@ -245,7 +245,7 @@ def process_and_execute_notebook(
         result.logs_bucket = operation_metadata.build.logs_bucket
 
         # Block and wait for the result
-        operation_result = operation.result()
+        operation_result = operation.result(timeout=86400)
 
         result.duration = datetime.datetime.now() - time_start
         result.is_pass = True

notebooks/community/sdk/sdk_pytorch_torchrun_custom_container_training_imagenet.ipynb

@@ -567,7 +567,7 @@
       "outputs": [],
       "source": [
         "%%writefile trainer/Dockerfile\n",
-        "FROM gcr.io/deeplearning-platform-release/pytorch-gpu.1-12\n",
+        "FROM gcr.io/deeplearning-platform-release/pytorch-gpu.1-13:m102\n",
         "\n",
         "RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \\\n",
         "    # Install reduction server plugin on GPU containers. google-fast-socket is\n",
@@ -588,16 +588,31 @@
         "RUN apt-get update -y && \\\n",
         "    apt-get install -y curl gnupg telnet nano net-tools iputils-ping\n",
         "\n",
+        "# Set ETCD version\n",
+        "ARG ETCD_VER=v2.3.0\n",
+        "# Choose either URL\n",
+        "ARG GOOGLE_URL=https://storage.googleapis.com/etcd\n",
+        "ARG GITHUB_URL=https://github.com/etcd-io/etcd/releases/download\n",
+        "# Set ETCD URL to download from\n",
+        "ARG DOWNLOAD_URL=$GOOGLE_URL\n",
         "\n",
+        "# Install ETCD\n",
+        "RUN mkdir -p /tmp/etcd-download-test && \\\n",
+        "    curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz && \\\n",
+        "    tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1 && \\\n",
+        "    rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz\n",
+        "\n",
+        "# Copy training application code\n",
         "COPY . /trainer\n",
         "\n",
         "WORKDIR /trainer\n",
         "\n",
+        "# Install dependencies\n",
         "RUN pip install -r requirements.txt\n",
         "\n",
         "RUN chmod 777 main.sh\n",
         "\n",
-        "# download data to the container\n",
+        "# Download data to the container\n",
         "RUN wget -q -P /trainer/data https://image-net.org/data/tiny-imagenet-200.zip\n",
         "RUN unzip -q /trainer/data/tiny-imagenet-200.zip\n",
         "RUN rm /trainer/data/tiny-imagenet-200.zip\n",
@@ -614,8 +629,8 @@
       "outputs": [],
       "source": [
         "%%writefile trainer/requirements.txt\n",
-        "torch==1.12.0\n",
-        "torchvision==0.13.0\n",
+        "torch==1.13.0\n",
+        "torchvision==0.14.0\n",
         "tensorboard==2.5.0\n",
         "protobuf==3.20.*\n",
         "python-etcd\n",
@@ -675,30 +690,17 @@
         "setup_etcd() {\n",
         "    HOST_IP=$1\n",
         "    # Start a local instane of ETCD v2 \n",
-        "    ETCD_VER=v2.3.0 #v3.5.6\n",
         "    export ETCD_ENABLE_V2=true\n",
         "    export ETCDCTL_API=2\n",
         "\n",
-        "    # choose either URL\n",
-        "    GOOGLE_URL=https://storage.googleapis.com/etcd\n",
-        "    GITHUB_URL=https://github.com/etcd-io/etcd/releases/download\n",
-        "    DOWNLOAD_URL=${GOOGLE_URL}\n",
-        "\n",
-        "    rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz\n",
-        "    rm -rf /tmp/etcd-download-test && mkdir -p /tmp/etcd-download-test\n",
-        "\n",
-        "    curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz\n",
-        "    tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1\n",
-        "    rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz\n",
-        "\n",
         "    /tmp/etcd-download-test/etcd --name s1 --data-dir /tmp/etcd-download-test/s1  \\\n",
         "    --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://$HOST_IP:2379 \\\n",
         "    --listen-peer-urls http://0.0.0.0:2380 --initial-advertise-peer-urls http://$HOST_IP:2380 \\\n",
         "    --initial-cluster s1=http://$HOST_IP:2380 --initial-cluster-token tkn \\\n",
         "    --initial-cluster-state new &> /tmp/etcd-download-test/node.log &\n",
         "\n",
-        "    sudo /tmp/etcd-download-test/etcd --version\n",
-        "    sudo /tmp/etcd-download-test/etcdctl --version\n",
+        "    /tmp/etcd-download-test/etcd --version\n",
+        "    /tmp/etcd-download-test/etcdctl --version\n",
         "}\n",
         "\n",
         "\n",

notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb

@@ -29,7 +29,7 @@
         "id": "l2mMvIUG9meX"
       },
       "source": [
-        "# Profile model training performance using Profiler\n",
+        "# Profile model training performance using Vertex AI TensorBoard Profiler\n",
         "\n",
         "<table align=\"left\">\n",
         "\n",
@@ -49,7 +49,7 @@
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
-        "  </td>                                                                                               \n",
+        "  </td>\n",
         "</table>"
       ]
     },
@@ -331,22 +331,16 @@
         "! gcloud projects add-iam-policy-binding $PROJECT_ID \\\n",
         "        --member=\"serviceAccount:$SERVICE_ACCOUNT\" \\\n",
         "        --role=\"roles/storage.admin\" \\\n",
-        "        --quiet"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "S_8_5jm-Gk6w"
-      },
-      "outputs": [],
-      "source": [
+        "        --quiet\n",
+        "\n",
         "# Grant AI Platform permission.\n",
         "! gcloud projects add-iam-policy-binding $PROJECT_ID \\\n",
         "        --member=\"serviceAccount:$SERVICE_ACCOUNT\" \\\n",
         "        --role=\"roles/aiplatform.user\" \\\n",
-        "        --quiet"
+        "        --quiet\n",
+        "\n",
+        "! gcloud projects get-iam-policy $PROJECT_ID \\\n",
+        "        --filter=bindings.members:serviceAccount:$SERVICE_ACCOUNT"
       ]
     },
     {
@@ -768,7 +762,7 @@
         "WORKDIR /root\n",
         "\n",
         "# Installs additional packages as you need.\n",
-        "RUN pip3 install google-cloud-aiplatform[cloud_profiler]\n",
+        "RUN pip3 install google-cloud-aiplatform[cloud_profiler]>=1.19.1\n",
         "\n",
         "# Copies the trainer code to the docker image.\n",
         "RUN mkdir /root/trainer\n",
@@ -798,7 +792,7 @@
         "IMAGE_NAME = \"tensorboard-custom-container\"\n",
         "IMAGE_URI = f\"{REGION}-docker.pkg.dev/{PROJECT_ID}/{DOCKER_REPOSITORY}/{IMAGE_NAME}\"\n",
         "\n",
-        "! gcloud builds submit --project {PROJECT_ID} --region={REGION} --tag {IMAGE_URI} --timeout=60m --quiet"
+        "! gcloud builds submit --project {PROJECT_ID} --region={REGION} --tag {IMAGE_URI} --timeout=3600s --quiet"
       ]
     },
     {