Ran linter

Updated Vertex AI Workbench link

.cloud-build/execute_changed_notebooks_helper.py

@@ -156,7 +156,7 @@ def _create_tag(filepath: str) -> str:
     return tag
 
 
-rate_limit = RateLimit(max_count=25, per=60, greedy=True)
+rate_limit = RateLimit(max_count=15, per=60, greedy=True)
 
 
 def process_and_execute_notebook(

community-content/pytorch_image_classification_distributed_data_parallel_training_with_vertex_sdk/trainer/requirements.txt

@@ -1,3 +1,3 @@
-torch==1.8.1
+torch==1.13.1
 torchvision==0.9.1
 tensorboard==2.5.0

notebooks/community/CODEOWNERS

@@ -41,3 +41,8 @@
 /notebooks/community/pipelines/google_cloud_pipeline_components_cloud_natural_language_pipeline.ipynb @Narwhalprime
 /notebooks/community/pipelines/google_cloud_pipeline_components_ready_to_go_text_classification_pipeline.ipynb @Narwhalprime
 /notebooks/community/feature_store/get_started_vertex_feature_store.ipynb @junkourata
+/notebooks/community/model_garden/model_garden_tfvision_image_classification.ipynb @genquan9
+/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion.ipynb @xiangxu-google
+/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_inpainting.ipynb @xiangxu-google
+/notebooks/community/model_garden/model_garden_pytorch_instructpix2pix.ipynb @xiangxu-google
+/notebooks/community/model_garden/model_garden_pytorch_controlnet.ipynb @xiangxu-google

notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb

@@ -560,6 +560,30 @@
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "set_service_account:pipelines"
+      },
+      "source": [
+        "#### Set service account access for Vertex AI Pipelines\n",
+        "\n",
+        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step -- you only need to run these once per service account."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_service_account:pipelines"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
+        "\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_xgboost_with_experiments.ipynb

@@ -557,6 +557,30 @@
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "set_service_account:pipelines"
+      },
+      "source": [
+        "#### Set service account access for Vertex AI Pipelines\n",
+        "\n",
+        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step -- you only need to run these once per service account."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 13,
+      "metadata": {
+        "id": "set_service_account:pipelines"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
+        "\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_xgboost_with_prediction.ipynb

@@ -557,6 +557,30 @@
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "set_service_account:pipelines"
+      },
+      "source": [
+        "#### Set service account access for Vertex AI Pipelines\n",
+        "\n",
+        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step -- you only need to run these once per service account."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 13,
+      "metadata": {
+        "id": "set_service_account:pipelines"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
+        "\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/community/model_garden/model_garden_pytorch_controlnet.ipynb

@@ -0,0 +1,622 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7d9bbf86da5e"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "99c1c3fc2ca5"
+      },
+      "source": [
+        "# Vertex AI Model Garden - ControlNet\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_controlnet.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_controlnet.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>                                                                                               <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_controlnet.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3de7470326a2"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates finetuning the [ControlNet](https://huggingface.co/lllyasviel/ControlNet) with the [fusing/fill50k](https://huggingface.co/datasets/fusing/fill50k) dataset and deploying the model on Vertex AI for online prediction.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "- Finetune the ControlNet model.\n",
+        "- Upload the model to [Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).\n",
+        "- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
+        "- Run online predictions for text-guided-image-to-image.\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264c07757582"
+      },
+      "source": [
+        "## Setup environment\n",
+        "\n",
+        "**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d73ffa0c0b83"
+      },
+      "source": [
+        "### Colab only"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2707b02ef5df"
+      },
+      "outputs": [],
+      "source": [
+        "!pip3 install --upgrade google-cloud-aiplatform"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b60a4d7100bf"
+      },
+      "outputs": [],
+      "source": [
+        "from google.colab import auth as google_auth\n",
+        "\n",
+        "google_auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fb671e75ca7b"
+      },
+      "source": [
+        "### Install dependencies"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dc8ee367fb42"
+      },
+      "outputs": [],
+      "source": [
+        "# Install gdown for downloading example training images.\n",
+        "!pip install gdown\n",
+        "# Install libs for generating conditioning images for ControlNet.\n",
+        "!pip install opencv-python"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5244aac3d929"
+      },
+      "source": [
+        "Restart the notebook kernel after installs."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "567212ff53a6"
+      },
+      "outputs": [],
+      "source": [
+        "import IPython\n",
+        "\n",
+        "app = IPython.Application.instance()\n",
+        "app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bb7adab99e41"
+      },
+      "source": [
+        "### Setup Google Cloud project\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6c460088b873"
+      },
+      "source": [
+        "Fill following variables for experiments environment:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "855d6b96f291"
+      },
+      "outputs": [],
+      "source": [
+        "# Cloud project id.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The region you want to launch jobs in.\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The Cloud Storage bucket for storing experiments output. Fill it without the 'gs://' prefix.\n",
+        "GCS_BUCKET = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e828eb320337"
+      },
+      "source": [
+        "Initialize Vertex AI API:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "12cd25839741"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=GCS_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2cc825514deb"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b42bd4fa2b2d"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-built training docker image. It contains training scripts and models.\n",
+        "TRAIN_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/pytorch-diffusers-train:latest\"\n",
+        "\n",
+        "# The pre-built serving docker image. It contains serving scripts and models.\n",
+        "SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/pytorch-diffusers-serve:latest\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0c250872074f"
+      },
+      "source": [
+        "### Define common functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "metadata": {
+        "id": "354da31189dc"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "from io import BytesIO\n",
+        "\n",
+        "import cv2\n",
+        "import numpy as np\n",
+        "import requests\n",
+        "from PIL import Image\n",
+        "\n",
+        "\n",
+        "def create_job_name(prefix):\n",
+        "    user = os.environ.get(\"USER\")\n",
+        "    now = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
+        "    job_name = f\"{prefix}-{user}-{now}\"\n",
+        "    return job_name\n",
+        "\n",
+        "\n",
+        "def download_image(url):\n",
+        "    response = requests.get(url)\n",
+        "    return Image.open(BytesIO(response.content))\n",
+        "\n",
+        "\n",
+        "def image_to_base64(image, format=\"JPEG\"):\n",
+        "    buffer = BytesIO()\n",
+        "    image.save(buffer, format=format)\n",
+        "    image_str = base64.b64encode(buffer.getvalue()).decode(\"utf-8\")\n",
+        "    return image_str\n",
+        "\n",
+        "\n",
+        "def base64_to_image(image_str):\n",
+        "    image = Image.open(BytesIO(base64.b64decode(image_str)))\n",
+        "    return image\n",
+        "\n",
+        "\n",
+        "def image_grid(imgs, rows=2, cols=2):\n",
+        "    w, h = imgs[0].size\n",
+        "    grid = Image.new(\"RGB\", size=(cols * w, rows * h))\n",
+        "    for i, img in enumerate(imgs):\n",
+        "        grid.paste(img, box=(i % cols * w, i // cols * h))\n",
+        "    return grid\n",
+        "\n",
+        "\n",
+        "def canny(image):\n",
+        "    image = np.array(image)\n",
+        "    image = cv2.Canny(image, 100, 200)\n",
+        "    image = image[:, :, None]\n",
+        "    image = np.concatenate([image, image, image], axis=2)\n",
+        "    image = Image.fromarray(image)\n",
+        "    return image\n",
+        "\n",
+        "\n",
+        "def deploy_model(model_id, task):\n",
+        "    model_name = \"controlnet\"\n",
+        "    endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n",
+        "    serving_env = {\n",
+        "        \"MODEL_ID\": model_id,\n",
+        "        \"TASK\": task,\n",
+        "    }\n",
+        "    # If the model_id is a GCS path, use artifact_uri to pass it to serving docker.\n",
+        "    artifact_uri = model_id if model_id.startswith(\"gs://\") else None\n",
+        "    model = aiplatform.Model.upload(\n",
+        "        display_name=model_name,\n",
+        "        serving_container_image_uri=SERVE_DOCKER_URI,\n",
+        "        serving_container_ports=[7080],\n",
+        "        serving_container_predict_route=\"/predictions/diffusers_serving\",\n",
+        "        serving_container_health_route=\"/ping\",\n",
+        "        serving_container_environment_variables=serving_env,\n",
+        "        artifact_uri=artifact_uri,\n",
+        "    )\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        machine_type=\"n1-standard-8\",\n",
+        "        accelerator_type=\"NVIDIA_TESLA_V100\",\n",
+        "        accelerator_count=1,\n",
+        "        deploy_request_timeout=1800,\n",
+        "    )\n",
+        "    return model, endpoint"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e70e3519ff8b"
+      },
+      "source": [
+        "## Finetune with fill50k dataset"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0dc65d8f0689"
+      },
+      "source": [
+        "This section uses the [fusing/fill50k](https://huggingface.co/datasets/fusing/fill50k) dataset to finetune the ControlNet model.\n",
+        "\n",
+        "The job will run on 1 A100 GPU and take ~7 hours to finish 1 epoch of training.\n",
+        "\n",
+        "The ControlNet model will be saved after the finetuning job finishs and it can be loaded to run inference later."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "65467b361315"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-trained stable diffusion model to be loaded.\n",
+        "stable_diffusion_model_id = \"runwayml/stable-diffusion-v1-5\"\n",
+        "# The datase id to be loaded.\n",
+        "dataset_id = \"fusing/fill50k\"\n",
+        "# The output path.\n",
+        "output_dir = f\"/gcs/{GCS_BUCKET}/controlnet/output\"\n",
+        "\n",
+        "# Worker pool spec.\n",
+        "machine_type = \"a2-highgpu-1g\"\n",
+        "num_nodes = 1\n",
+        "gpu_type = \"NVIDIA_TESLA_A100\"\n",
+        "num_gpus = 1\n",
+        "\n",
+        "# Setup training job.\n",
+        "job_name = create_job_name(\"controlnet\")\n",
+        "job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=job_name,\n",
+        "    container_uri=TRAIN_DOCKER_URI,\n",
+        ")\n",
+        "\n",
+        "# Pass training arguments and launch job.\n",
+        "# See https://github.com/huggingface/diffusers/blob/main/examples/controlnet/train_controlnet.py\n",
+        "# for a full list of training arguments.\n",
+        "model = job.run(\n",
+        "    args=[\n",
+        "        \"controlnet/train_controlnet.py\",\n",
+        "        \"--tracker_project_name=train_controlnet\",\n",
+        "        f\"--pretrained_model_name_or_path={stable_diffusion_model_id}\",\n",
+        "        f\"--output_dir={output_dir}\",\n",
+        "        f\"--dataset_name={dataset_id}\",\n",
+        "        \"--resolution=512\",\n",
+        "        \"--learning_rate=1e-5\",\n",
+        "        \"--train_batch_size=2\",\n",
+        "    ],\n",
+        "    replica_count=num_nodes,\n",
+        "    machine_type=machine_type,\n",
+        "    accelerator_type=gpu_type,\n",
+        "    accelerator_count=num_gpus,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bf7f82732e61"
+      },
+      "source": [
+        "## Upload and Deploy models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1cc26e68d7b0"
+      },
+      "source": [
+        "This section uploads the model to Model Registry and deploys it on the Endpoint.\n",
+        "\n",
+        "The model deployment step will take ~15 minutes to complete."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cd7b56421392"
+      },
+      "source": [
+        "### Pre-trained canny model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6d331b1ea337"
+      },
+      "source": [
+        "Deploy the pre-trained [lllyasviel/sd-controlnet-canny](https://huggingface.co/lllyasviel/sd-controlnet-canny) model for the text-guided image-to-image task. When deployed on one V100 GPU, the averaged inference time of a request is ~15 seconds."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bf55e38815dc"
+      },
+      "outputs": [],
+      "source": [
+        "model, endpoint = deploy_model(\n",
+        "    model_id=\"lllyasviel/sd-controlnet-canny\", task=\"controlnet\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4ab04da3ec9a"
+      },
+      "outputs": [],
+      "source": [
+        "init_image = download_image(\n",
+        "    \"https://huggingface.co/takuma104/controlnet_dev/resolve/main/gen_compare/output_images/diffusers/output_bird_canny_1.png\"\n",
+        ")\n",
+        "display(init_image)\n",
+        "image = canny(init_image)\n",
+        "display(image)\n",
+        "\n",
+        "instances = [\n",
+        "    {\n",
+        "        \"prompt\": \"bird\",\n",
+        "        \"image\": image_to_base64(image),\n",
+        "    },\n",
+        "]\n",
+        "response = endpoint.predict(instances=instances)\n",
+        "images = [base64_to_image(image) for image in response.predictions]\n",
+        "display(images[0])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "af21a3cff1e0"
+      },
+      "source": [
+        "Clean up resources:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "911406c1561e"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete models.\n",
+        "model.delete()\n",
+        "\n",
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c1e51f764a60"
+      },
+      "source": [
+        "### Custom finetuned fill50k model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fa686a54047c"
+      },
+      "source": [
+        "Deploy the finetuned fill50k model above for the text-guided image-to-image task. When deployed on one V100 GPU, the averaged inference time of a request is ~15 seconds."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "65e32356fbd1"
+      },
+      "outputs": [],
+      "source": [
+        "model, endpoint = deploy_model(\n",
+        "    model_id=f\"gs://{GCS_BUCKET}/controlnet/output\", task=\"image-to-image\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "83a50fd4a1ed"
+      },
+      "outputs": [],
+      "source": [
+        "init_image = download_image(\n",
+        "    \"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/controlnet_training/conditioning_image_1.png\"\n",
+        ")\n",
+        "display(init_image)\n",
+        "\n",
+        "instances = [\n",
+        "    {\n",
+        "        \"prompt\": \"red circle with green background\",\n",
+        "        \"image\": image_to_base64(init_image, format=\"PNG\"),\n",
+        "    },\n",
+        "]\n",
+        "response = endpoint.predict(instances=instances)\n",
+        "images = [base64_to_image(image) for image in response.predictions]\n",
+        "display(images[0])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ed3795d474b9"
+      },
+      "source": [
+        "Clean up resources:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b53b883257b4"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete models.\n",
+        "model.delete()\n",
+        "\n",
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pytorch_controlnet.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pytorch_instructpix2pix.ipynb

@@ -0,0 +1,415 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7d9bbf86da5e"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2bd716bf3e39"
+      },
+      "source": [
+        "# Vertex AI Model Garden - InstructPix2Pix\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_instructpix2pix.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_instructpix2pix.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>                                                                                               <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_instructpix2pix.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d8cd12648da4"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates deploying the pre-trained [InstructPix2Pix](https://huggingface.co/timbrooks/instruct-pix2pix) model on Vertex AI for online prediction.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "- Upload the model to [Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).\n",
+        "- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
+        "- Run online predictions for text-guided image-to-image.\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264c07757582"
+      },
+      "source": [
+        "## Setup environment\n",
+        "\n",
+        "**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d73ffa0c0b83"
+      },
+      "source": [
+        "### Colab only"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2707b02ef5df"
+      },
+      "outputs": [],
+      "source": [
+        "!pip3 install --upgrade google-cloud-aiplatform"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b60a4d7100bf"
+      },
+      "outputs": [],
+      "source": [
+        "from google.colab import auth as google_auth\n",
+        "\n",
+        "google_auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0f826ff482a2"
+      },
+      "source": [
+        "### Setup Google Cloud project\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8958ebc71868"
+      },
+      "source": [
+        "Fill following variables for experiments environment:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9db30f827a65"
+      },
+      "outputs": [],
+      "source": [
+        "# Cloud project id.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The region you want to launch jobs in.\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The Cloud Storage bucket for storing experiments output. Fill it without the 'gs://' prefix.\n",
+        "GCS_BUCKET = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "92f16e22c20b"
+      },
+      "source": [
+        "Initialize Vertex AI API:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1680c257acfb"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=GCS_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6ca48b699d17"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "de9882ea89ea"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-built serving docker image. It contains serving scripts and models.\n",
+        "SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/pytorch-diffusers-serve:latest\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "10188266a5cd"
+      },
+      "source": [
+        "### Define common functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cac4478ae098"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "import os\n",
+        "from datetime import datetime\n",
+        "from io import BytesIO\n",
+        "\n",
+        "import requests\n",
+        "from google.cloud import aiplatform\n",
+        "from PIL import Image\n",
+        "\n",
+        "\n",
+        "def create_job_name(prefix):\n",
+        "    user = os.environ.get(\"USER\")\n",
+        "    now = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
+        "    job_name = f\"{prefix}-{user}-{now}\"\n",
+        "    return job_name\n",
+        "\n",
+        "\n",
+        "def download_image(url):\n",
+        "    response = requests.get(url)\n",
+        "    return Image.open(BytesIO(response.content))\n",
+        "\n",
+        "\n",
+        "def image_to_base64(image, format=\"JPEG\"):\n",
+        "    buffer = BytesIO()\n",
+        "    image.save(buffer, format=format)\n",
+        "    image_str = base64.b64encode(buffer.getvalue()).decode(\"utf-8\")\n",
+        "    return image_str\n",
+        "\n",
+        "\n",
+        "def base64_to_image(image_str):\n",
+        "    image = Image.open(BytesIO(base64.b64decode(image_str)))\n",
+        "    return image\n",
+        "\n",
+        "\n",
+        "def image_grid(imgs, rows=2, cols=2):\n",
+        "    w, h = imgs[0].size\n",
+        "    grid = Image.new(\"RGB\", size=(cols * w, rows * h))\n",
+        "    for i, img in enumerate(imgs):\n",
+        "        grid.paste(img, box=(i % cols * w, i // cols * h))\n",
+        "    return grid\n",
+        "\n",
+        "\n",
+        "def deploy_model(model_id, task):\n",
+        "    model_name = \"instruct-pix2pix\"\n",
+        "    endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n",
+        "    serving_env = {\n",
+        "        \"MODEL_ID\": model_id,\n",
+        "        \"TASK\": task,\n",
+        "    }\n",
+        "    # If the model_id is a GCS path, use artifact_uri to pass it to serving docker.\n",
+        "    artifact_uri = model_id if model_id.startswith(\"gs://\") else None\n",
+        "    model = aiplatform.Model.upload(\n",
+        "        display_name=model_name,\n",
+        "        serving_container_image_uri=SERVE_DOCKER_URI,\n",
+        "        serving_container_ports=[7080],\n",
+        "        serving_container_predict_route=\"/predictions/diffusers_serving\",\n",
+        "        serving_container_health_route=\"/ping\",\n",
+        "        serving_container_environment_variables=serving_env,\n",
+        "        artifact_uri=artifact_uri,\n",
+        "    )\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        machine_type=\"n1-standard-8\",\n",
+        "        accelerator_type=\"NVIDIA_TESLA_V100\",\n",
+        "        accelerator_count=1,\n",
+        "        deploy_request_timeout=1800,\n",
+        "    )\n",
+        "    return model, endpoint"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d2d72ecdb8c9"
+      },
+      "source": [
+        "## Upload and deploy models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9448c5f545fa"
+      },
+      "source": [
+        "This section uploads the pre-trained model to Model Registry and deploys it on the Endpoint.\n",
+        "\n",
+        "The model deployment step will take ~15 minutes to complete."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c277da31bde6"
+      },
+      "source": [
+        "### Text-guided image-to-image"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a5a86996222c"
+      },
+      "source": [
+        "Deploy the InstructPix2Pix model for the text-guided image-to-image task.\n",
+        "\n",
+        "Once deployed, you can send prompts to the endpoint to generated images.\n",
+        "\n",
+        "When deployed on one V100 GPU, the averaged inference time of a request is ~15 seconds."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b4b46c28d8b1"
+      },
+      "outputs": [],
+      "source": [
+        "model, endpoint = deploy_model(\n",
+        "    model_id=\"timbrooks/instruct-pix2pix\", task=\"instruct-pix2pix\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6be655247cb1"
+      },
+      "outputs": [],
+      "source": [
+        "init_image = download_image(\n",
+        "    \"https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/mountain.png\"\n",
+        ")\n",
+        "display(init_image)\n",
+        "instances = [\n",
+        "    {\n",
+        "        \"prompt\": \"Add fire to the mountain\",\n",
+        "        \"image\": image_to_base64(init_image),\n",
+        "    },\n",
+        "]\n",
+        "response = endpoint.predict(instances=instances)\n",
+        "images = [base64_to_image(image) for image in response.predictions]\n",
+        "display(images[0])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "db7ffebdb4be"
+      },
+      "source": [
+        "### Clean up resources"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2ccf3714dbe9"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete models.\n",
+        "model.delete()\n",
+        "\n",
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pytorch_instructpix2pix.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pytorch_stable_diffusion.ipynb

@@ -0,0 +1,648 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7d9bbf86da5e"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "99c1c3fc2ca5"
+      },
+      "source": [
+        "# Vertex AI Model Garden - Stable Diffusion V1.5\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>                                                                                               <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3de7470326a2"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates finetuning [runwayml/stable-diffusion-v1-5](https://huggingface.co/runwayml/stable-diffusion-v1-5) with [Dreambooth](https://huggingface.co/docs/diffusers/training/dreambooth) and deploying it on Vertex AI for online prediction.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "- Finetune the stable-diffusion-v1.5 model with [Dreambooth](https://huggingface.co/docs/diffusers/training/dreambooth).\n",
+        "- Upload the model to [Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).\n",
+        "- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
+        "- Run online predictions for text-to-image and text-guided-image-to-image.\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264c07757582"
+      },
+      "source": [
+        "## Setup environment\n",
+        "\n",
+        "**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d73ffa0c0b83"
+      },
+      "source": [
+        "### Colab only"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2707b02ef5df"
+      },
+      "outputs": [],
+      "source": [
+        "!pip3 install --upgrade google-cloud-aiplatform"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b60a4d7100bf"
+      },
+      "outputs": [],
+      "source": [
+        "from google.colab import auth as google_auth\n",
+        "\n",
+        "google_auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fb671e75ca7b"
+      },
+      "source": [
+        "### Install dependencies"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dc8ee367fb42"
+      },
+      "outputs": [],
+      "source": [
+        "# Install gdown for downloading example training images.\n",
+        "!pip install gdown\n",
+        "# Install gsutil for downloading/uploading data from/to Cloud Storage buckets.\n",
+        "!pip install gsutil"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5244aac3d929"
+      },
+      "source": [
+        "Restart the notebook kernel after installs."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "567212ff53a6"
+      },
+      "outputs": [],
+      "source": [
+        "import IPython\n",
+        "\n",
+        "app = IPython.Application.instance()\n",
+        "app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bb7adab99e41"
+      },
+      "source": [
+        "### Setup Google Cloud project\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6c460088b873"
+      },
+      "source": [
+        "Fill following variables for experiments environment:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "855d6b96f291"
+      },
+      "outputs": [],
+      "source": [
+        "# Cloud project id.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The region you want to launch jobs in.\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The Cloud Storage bucket for storing experiments output. Fill it without the 'gs://' prefix.\n",
+        "GCS_BUCKET = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e828eb320337"
+      },
+      "source": [
+        "Initialize Vertex-AI API:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "12cd25839741"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=GCS_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2cc825514deb"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b42bd4fa2b2d"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-built training docker image. It contains training scripts and models.\n",
+        "TRAIN_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/pytorch-diffusers-train:latest\"\n",
+        "\n",
+        "# The pre-built serving docker image. It contains serving scripts and models.\n",
+        "SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/pytorch-diffusers-serve:latest\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0c250872074f"
+      },
+      "source": [
+        "### Define common functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "354da31189dc"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "import os\n",
+        "from datetime import datetime\n",
+        "from io import BytesIO\n",
+        "\n",
+        "import requests\n",
+        "from google.cloud import aiplatform\n",
+        "from PIL import Image\n",
+        "\n",
+        "\n",
+        "def create_job_name(prefix):\n",
+        "    user = os.environ.get(\"USER\")\n",
+        "    now = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
+        "    job_name = f\"{prefix}-{user}-{now}\"\n",
+        "    return job_name\n",
+        "\n",
+        "\n",
+        "def download_image(url):\n",
+        "    response = requests.get(url)\n",
+        "    return Image.open(BytesIO(response.content))\n",
+        "\n",
+        "\n",
+        "def image_to_base64(image, format=\"JPEG\"):\n",
+        "    buffer = BytesIO()\n",
+        "    image.save(buffer, format=format)\n",
+        "    image_str = base64.b64encode(buffer.getvalue()).decode(\"utf-8\")\n",
+        "    return image_str\n",
+        "\n",
+        "\n",
+        "def base64_to_image(image_str):\n",
+        "    image = Image.open(BytesIO(base64.b64decode(image_str)))\n",
+        "    return image\n",
+        "\n",
+        "\n",
+        "def image_grid(imgs, rows=2, cols=2):\n",
+        "    w, h = imgs[0].size\n",
+        "    grid = Image.new(\"RGB\", size=(cols * w, rows * h))\n",
+        "    for i, img in enumerate(imgs):\n",
+        "        grid.paste(img, box=(i % cols * w, i // cols * h))\n",
+        "    return grid\n",
+        "\n",
+        "\n",
+        "def deploy_model(model_id, task):\n",
+        "    model_name = \"stable-diffusion-v1\"\n",
+        "    endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-{task}-endpoint\")\n",
+        "    serving_env = {\n",
+        "        \"MODEL_ID\": model_id,\n",
+        "        \"TASK\": task,\n",
+        "    }\n",
+        "    # If the model_id is a GCS path, use artifact_uri to pass it to serving docker.\n",
+        "    artifact_uri = model_id if model_id.startswith(\"gs://\") else None\n",
+        "    model = aiplatform.Model.upload(\n",
+        "        display_name=model_name,\n",
+        "        serving_container_image_uri=SERVE_DOCKER_URI,\n",
+        "        serving_container_ports=[7080],\n",
+        "        serving_container_predict_route=\"/predictions/diffusers_serving\",\n",
+        "        serving_container_health_route=\"/ping\",\n",
+        "        serving_container_environment_variables=serving_env,\n",
+        "        artifact_uri=artifact_uri,\n",
+        "    )\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        machine_type=\"n1-standard-8\",\n",
+        "        accelerator_type=\"NVIDIA_TESLA_V100\",\n",
+        "        accelerator_count=1,\n",
+        "        deploy_request_timeout=1800,\n",
+        "    )\n",
+        "    return model, endpoint"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e70e3519ff8b"
+      },
+      "source": [
+        "## Finetune with Dreambooth"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0dc65d8f0689"
+      },
+      "source": [
+        "This section uses [dreambooth](https://dreambooth.github.io/) to finetune the [stable-diffusion-v1.5](https://huggingface.co/runwayml/stable-diffusion-v1-5) model with [5 dog images](https://drive.google.com/drive/folders/1BO_dyz-p65qhBRRMRA4TbZ8qW4rB99JZ) to personalize the text-to-image model.\n",
+        "\n",
+        "It finetunes both text encoder and unet of the stable diffusion model up to 800 steps. The whole finetuning job takes 30 minutes to finish using 1 A100 GPU.\n",
+        "\n",
+        "The full model will be saved after the finetuning job finishs and it can be loaded by the [StableDiffusionPipeline](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/text2img) to run inference."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "34048707df5c"
+      },
+      "outputs": [],
+      "source": [
+        "# Download example training images.\n",
+        "!gdown --folder https://drive.google.com/drive/folders/1BO_dyz-p65qhBRRMRA4TbZ8qW4rB99JZ\n",
+        "\n",
+        "# Upload data to Cloud Storage bucket.\n",
+        "!gsutil -m cp -r dog/* gs://{GCS_BUCKET}/dreambooth/dog/\n",
+        "!gsutil -m cp -r dog/* gs://{GCS_BUCKET}/dreambooth/dog_class/"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "969cfeb79317"
+      },
+      "source": [
+        "**NOTE**: If the upload step fails due to lacking of permission, you need to [grant the Storage Object Admin role](https://cloud.google.com/storage/docs/access-control/using-iam-permissions) for the Cloud account of the notebook."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "65467b361315"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-trained model to be loaded.\n",
+        "model_id = \"runwayml/stable-diffusion-v1-5\"\n",
+        "\n",
+        "# Input and output path.\n",
+        "instance_dir = f\"/gcs/{GCS_BUCKET}/dreambooth/dog\"\n",
+        "class_dir = f\"/gcs/{GCS_BUCKET}/dreambooth/dog_class\"\n",
+        "output_dir = f\"/gcs/{GCS_BUCKET}/dreambooth/output\"\n",
+        "\n",
+        "# Worker pool spec.\n",
+        "machine_type = \"a2-highgpu-1g\"\n",
+        "num_nodes = 1\n",
+        "gpu_type = \"NVIDIA_TESLA_A100\"\n",
+        "num_gpus = 1\n",
+        "\n",
+        "# Setup training job.\n",
+        "job_name = create_job_name(\"dreambooth-stable-diffusion\")\n",
+        "job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=job_name,\n",
+        "    container_uri=TRAIN_DOCKER_URI,\n",
+        ")\n",
+        "\n",
+        "# Pass training arguments and launch job.\n",
+        "# See https://github.com/huggingface/diffusers/blob/v0.14.0/examples/dreambooth/train_dreambooth.py#L75\n",
+        "# for a full list of training arguments.\n",
+        "model = job.run(\n",
+        "    args=[\n",
+        "        \"dreambooth/train_dreambooth.py\",\n",
+        "        f\"--pretrained_model_name_or_path={model_id}\",\n",
+        "        \"--train_text_encoder\",\n",
+        "        f\"--instance_data_dir={instance_dir}\",\n",
+        "        f\"--class_data_dir={class_dir}\",\n",
+        "        f\"--output_dir={output_dir}\",\n",
+        "        \"--with_prior_preservation\",\n",
+        "        \"--prior_loss_weight=1.0\",\n",
+        "        \"--instance_prompt='a photo of sks dog'\",\n",
+        "        \"--class_prompt='a photo of dog'\",\n",
+        "        \"--resolution=512\",\n",
+        "        \"--train_batch_size=1\",\n",
+        "        \"--gradient_checkpointing\",\n",
+        "        \"--learning_rate=2e-6\",\n",
+        "        \"--lr_scheduler=constant\",\n",
+        "        \"--lr_warmup_steps=0\",\n",
+        "        \"--num_class_images=200\",\n",
+        "        \"--max_train_steps=800\",\n",
+        "    ],\n",
+        "    replica_count=num_nodes,\n",
+        "    machine_type=machine_type,\n",
+        "    accelerator_type=gpu_type,\n",
+        "    accelerator_count=num_gpus,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bf7f82732e61"
+      },
+      "source": [
+        "## Upload and Deploy models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1cc26e68d7b0"
+      },
+      "source": [
+        "This section uploads the model to Model Registry and deploys it on the Endpoint.\n",
+        "\n",
+        "The model deployment step will take ~15 minutes to complete."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cd7b56421392"
+      },
+      "source": [
+        "### Text-to-image"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6d331b1ea337"
+      },
+      "source": [
+        "Deploy the stable diffusion model for the text-to-image task.\n",
+        "\n",
+        "Once deployed, you can send a batch of text prompts to the endpoint to generated images.\n",
+        "\n",
+        "When deployed on one V100 GPU, the averaged inference time of a request is ~15 seconds."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bf55e38815dc"
+      },
+      "outputs": [],
+      "source": [
+        "# Set the model_id to a GCS path, like \"gs://<GCS_BUCKET>/dreambooth/output\", to load the dreambooth finetuned model above.\n",
+        "model, endpoint = deploy_model(\n",
+        "    model_id=\"runwayml/stable-diffusion-v1-5\", task=\"text-to-image\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4ab04da3ec9a"
+      },
+      "outputs": [],
+      "source": [
+        "instances = [\n",
+        "    {\"prompt\": \"a squirrel in Picasso style\"},\n",
+        "    {\"prompt\": \"a dog in Picasso style\"},\n",
+        "    {\"prompt\": \"a cat in Picasso style\"},\n",
+        "    {\"prompt\": \"a deer in Picasso style\"},\n",
+        "]\n",
+        "response = endpoint.predict(instances=instances)\n",
+        "images = [base64_to_image(image) for image in response.predictions]\n",
+        "image_grid(images)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "af21a3cff1e0"
+      },
+      "source": [
+        "Clean up resources:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "911406c1561e"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete models.\n",
+        "model.delete()\n",
+        "\n",
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c1e51f764a60"
+      },
+      "source": [
+        "### Text-guided image-to-image"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fa686a54047c"
+      },
+      "source": [
+        "Deploy the stable diffusion model for the text-guided image-to-image task."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "65e32356fbd1"
+      },
+      "outputs": [],
+      "source": [
+        "# Set the model_id to a GCS path, like \"gs://<GCS_BUCKET>/dreambooth/output\", to load the dreambooth finetuned model above.\n",
+        "model, endpoint = deploy_model(\n",
+        "    model_id=\"runwayml/stable-diffusion-v1-5\", task=\"image-to-image\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "83a50fd4a1ed"
+      },
+      "outputs": [],
+      "source": [
+        "init_image = download_image(\n",
+        "    \"https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg\"\n",
+        ")\n",
+        "display(init_image)\n",
+        "instances = [\n",
+        "    {\n",
+        "        \"prompt\": \"A fantasy landscape, trending on artstation\",\n",
+        "        \"image\": image_to_base64(init_image),\n",
+        "    },\n",
+        "]\n",
+        "response = endpoint.predict(instances=instances)\n",
+        "images = [base64_to_image(image) for image in response.predictions]\n",
+        "display(images[0])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ed3795d474b9"
+      },
+      "source": [
+        "Clean up resources:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b53b883257b4"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete models.\n",
+        "model.delete()\n",
+        "\n",
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pytorch_stable_diffusion.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_inpainting.ipynb

@@ -0,0 +1,577 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7d9bbf86da5e"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1e9c07efb6ac"
+      },
+      "source": [
+        "# Vertex AI Model Garden - Stable Diffusion Inpainting\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_inpainting.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_inpainting.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>                                                                                               <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_inpainting.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cd8433ec804a"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates finetuning [runwayml/stable-diffusion-inpainting](https://huggingface.co/runwayml/stable-diffusion-inpainting) with [Dreambooth](https://huggingface.co/docs/diffusers/training/dreambooth) and deploying it on Vertex-AI for online prediction.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "- Finetune the stable-diffusion-inpainting model with [Dreambooth](https://huggingface.co/docs/diffusers/training/dreambooth).\n",
+        "- Upload the model to [Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).\n",
+        "- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
+        "- Run online predictions for image-inpainting.\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "264c07757582"
+      },
+      "source": [
+        "## Setup environment\n",
+        "\n",
+        "**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d73ffa0c0b83"
+      },
+      "source": [
+        "### Colab only"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2707b02ef5df"
+      },
+      "outputs": [],
+      "source": [
+        "!pip3 install --upgrade google-cloud-aiplatform"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b60a4d7100bf"
+      },
+      "outputs": [],
+      "source": [
+        "from google.colab import auth as google_auth\n",
+        "\n",
+        "google_auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fb671e75ca7b"
+      },
+      "source": [
+        "### Install dependencies"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dc8ee367fb42"
+      },
+      "outputs": [],
+      "source": [
+        "# Install gdown for downloading example training images.\n",
+        "!pip install gdown\n",
+        "# Install gsutil for downloading/uploading data from/to Cloud Storage buckets.\n",
+        "!pip install gsutil"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5244aac3d929"
+      },
+      "source": [
+        "Restart the notebook kernel after installs."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "567212ff53a6"
+      },
+      "outputs": [],
+      "source": [
+        "import IPython\n",
+        "\n",
+        "app = IPython.Application.instance()\n",
+        "app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bb7adab99e41"
+      },
+      "source": [
+        "### Setup Google Cloud project\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "\n",
+        "1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6c460088b873"
+      },
+      "source": [
+        "Fill following variables for experiments environment:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "855d6b96f291"
+      },
+      "outputs": [],
+      "source": [
+        "# Cloud project id.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The region you want to launch jobs in.\n",
+        "REGION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# The Cloud Storage bucket for storing experiments output. Fill it without the 'gs://' prefix.\n",
+        "GCS_BUCKET = \"\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e828eb320337"
+      },
+      "source": [
+        "Initialize Vertex-AI API:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "12cd25839741"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=GCS_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2cc825514deb"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b42bd4fa2b2d"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-built training docker image. It contains training scripts and models.\n",
+        "TRAIN_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/pytorch-diffusers-train:latest\"\n",
+        "\n",
+        "# The pre-built serving docker image. It contains serving scripts and models.\n",
+        "SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/pytorch-diffusers-serve:latest\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0c250872074f"
+      },
+      "source": [
+        "### Define common functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 2,
+      "metadata": {
+        "id": "8759e624ebc0"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "import os\n",
+        "from datetime import datetime\n",
+        "from io import BytesIO\n",
+        "\n",
+        "import requests\n",
+        "from google.cloud import aiplatform\n",
+        "from PIL import Image\n",
+        "\n",
+        "\n",
+        "def create_job_name(prefix):\n",
+        "    user = os.environ.get(\"USER\")\n",
+        "    now = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
+        "    job_name = f\"{prefix}-{user}-{now}\"\n",
+        "    return job_name\n",
+        "\n",
+        "\n",
+        "def download_image(url):\n",
+        "    response = requests.get(url)\n",
+        "    return Image.open(BytesIO(response.content))\n",
+        "\n",
+        "\n",
+        "def image_to_base64(image, format=\"JPEG\"):\n",
+        "    buffer = BytesIO()\n",
+        "    image.save(buffer, format=format)\n",
+        "    image_str = base64.b64encode(buffer.getvalue()).decode(\"utf-8\")\n",
+        "    return image_str\n",
+        "\n",
+        "\n",
+        "def base64_to_image(image_str):\n",
+        "    image = Image.open(BytesIO(base64.b64decode(image_str)))\n",
+        "    return image\n",
+        "\n",
+        "\n",
+        "def image_grid(imgs, rows=2, cols=2):\n",
+        "    w, h = imgs[0].size\n",
+        "    grid = Image.new(\"RGB\", size=(cols * w, rows * h))\n",
+        "    for i, img in enumerate(imgs):\n",
+        "        grid.paste(img, box=(i % cols * w, i // cols * h))\n",
+        "    return grid\n",
+        "\n",
+        "\n",
+        "def deploy_model(model_id, task):\n",
+        "    model_name = \"stable-diffusion-inpainting\"\n",
+        "    endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n",
+        "    serving_env = {\n",
+        "        \"MODEL_ID\": model_id,\n",
+        "        \"TASK\": task,\n",
+        "    }\n",
+        "    # If the model_id is a GCS path, use artifact_uri to pass it to serving docker.\n",
+        "    artifact_uri = model_id if model_id.startswith(\"gs://\") else None\n",
+        "    model = aiplatform.Model.upload(\n",
+        "        display_name=model_name,\n",
+        "        serving_container_image_uri=SERVE_DOCKER_URI,\n",
+        "        serving_container_ports=[7080],\n",
+        "        serving_container_predict_route=\"/predictions/diffusers_serving\",\n",
+        "        serving_container_health_route=\"/ping\",\n",
+        "        serving_container_environment_variables=serving_env,\n",
+        "        artifact_uri=artifact_uri,\n",
+        "    )\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        machine_type=\"n1-standard-8\",\n",
+        "        accelerator_type=\"NVIDIA_TESLA_V100\",\n",
+        "        accelerator_count=1,\n",
+        "        deploy_request_timeout=1800,\n",
+        "    )\n",
+        "    return model, endpoint"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e70e3519ff8b"
+      },
+      "source": [
+        "## Finetune with Dreambooth"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f560edbf96c6"
+      },
+      "source": [
+        "This section uses [dreambooth](https://dreambooth.github.io/) to finetune the [stable-diffusion-inpainting](https://huggingface.co/runwayml/stable-diffusion-inpainting) model with [5 dog images](https://drive.google.com/drive/folders/1BO_dyz-p65qhBRRMRA4TbZ8qW4rB99JZ) to personalize the model.\n",
+        "\n",
+        "It finetunes both text encoder and unet of the stable diffusion model up to 800 steps. The whole finetuning job takes 30 minutes to finish using 1 A100 GPU.\n",
+        "\n",
+        "The full model will be saved after the finetuning job finishs and it can be loaded by the [StableDiffusionInpaintPipeline](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/inpaint) to run inference."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "34048707df5c"
+      },
+      "outputs": [],
+      "source": [
+        "# Download example training images.\n",
+        "!gdown --folder https://drive.google.com/drive/folders/1BO_dyz-p65qhBRRMRA4TbZ8qW4rB99JZ\n",
+        "\n",
+        "# Upload data to Cloud Storage bucket.\n",
+        "!gsutil -m cp -r dog/* gs://{GCS_BUCKET}/dreambooth/dog/\n",
+        "!gsutil -m cp -r dog/* gs://{GCS_BUCKET}/dreambooth/dog_class/"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "969cfeb79317"
+      },
+      "source": [
+        "**NOTE**: If the upload step fails due to lacking of permission, you need to [grant the Storage Object Admin role](https://cloud.google.com/storage/docs/access-control/using-iam-permissions) for the Cloud account of the notebook."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f6d5a05592e1"
+      },
+      "outputs": [],
+      "source": [
+        "# The pre-trained model to be loaded.\n",
+        "model_id = \"runwayml/stable-diffusion-inpainting\"\n",
+        "\n",
+        "# Input and output path.\n",
+        "instance_dir = f\"/gcs/{GCS_BUCKET}/dreambooth/dog\"\n",
+        "class_dir = f\"/gcs/{GCS_BUCKET}/dreambooth/dog_class\"\n",
+        "output_dir = f\"/gcs/{GCS_BUCKET}/dreambooth/output\"\n",
+        "\n",
+        "# Worker pool spec.\n",
+        "machine_type = \"a2-highgpu-1g\"\n",
+        "num_nodes = 1\n",
+        "gpu_type = \"NVIDIA_TESLA_A100\"\n",
+        "num_gpus = 1\n",
+        "\n",
+        "# Setup training job.\n",
+        "job_name = create_job_name(\"dreambooth-stable-diffusion-inpainting\")\n",
+        "job = aiplatform.CustomContainerTrainingJob(\n",
+        "    display_name=job_name,\n",
+        "    container_uri=TRAIN_DOCKER_URI,\n",
+        ")\n",
+        "\n",
+        "# Pass training arguments and launch job.\n",
+        "# See https://github.com/huggingface/diffusers/blob/v0.14.0/examples/research_projects/dreambooth_inpaint/train_dreambooth_inpaint.py#L83\n",
+        "# for a full list of training arguments.\n",
+        "model = job.run(\n",
+        "    args=[\n",
+        "        \"research_projects/dreambooth_inpaint/train_dreambooth_inpaint.py\",\n",
+        "        f\"--pretrained_model_name_or_path={model_id}\",\n",
+        "        \"--train_text_encoder\",\n",
+        "        f\"--instance_data_dir={instance_dir}\",\n",
+        "        f\"--class_data_dir={class_dir}\",\n",
+        "        f\"--output_dir={output_dir}\",\n",
+        "        \"--with_prior_preservation\",\n",
+        "        \"--prior_loss_weight=1.0\",\n",
+        "        \"--instance_prompt='a photo of sks dog'\",\n",
+        "        \"--class_prompt='a photo of dog'\",\n",
+        "        \"--resolution=512\",\n",
+        "        \"--train_batch_size=1\",\n",
+        "        \"--gradient_checkpointing\",\n",
+        "        \"--learning_rate=2e-6\",\n",
+        "        \"--lr_scheduler=constant\",\n",
+        "        \"--lr_warmup_steps=0\",\n",
+        "        \"--num_class_images=200\",\n",
+        "        \"--max_train_steps=800\",\n",
+        "    ],\n",
+        "    replica_count=num_nodes,\n",
+        "    machine_type=machine_type,\n",
+        "    accelerator_type=gpu_type,\n",
+        "    accelerator_count=num_gpus,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "90d3c379090e"
+      },
+      "source": [
+        "## Upload and deploy models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1cc26e68d7b0"
+      },
+      "source": [
+        "This section uploads the model to Model Registry and deploys it on the Endpoint.\n",
+        "\n",
+        "The model deployment step will take ~15 minutes to complete."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b8bb7d198315"
+      },
+      "source": [
+        "### Image-inpainting"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "79b66382f849"
+      },
+      "source": [
+        "Deploy the stable diffusion model for the image-inpainting task.\n",
+        "\n",
+        "Once deployed, you can send prompts to the endpoint to generated images.\n",
+        "\n",
+        "When deployed on one V100 GPU, the averaged inference time of a request is ~15 seconds."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a881564da1d8"
+      },
+      "outputs": [],
+      "source": [
+        "# Set the model_id to a GCS path, like \"gs://<GCS_BUCKET>/dreambooth/output\", to load the dreambooth finetuned model above.\n",
+        "model, endpoint = deploy_model(\n",
+        "    model_id=\"runwayml/stable-diffusion-inpainting\", task=\"image-inpainting\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ca1761afb66f"
+      },
+      "outputs": [],
+      "source": [
+        "img_url = \"https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/bertrand-gabioud-CpuFzIsHYJ0.png\"\n",
+        "mask_url = \"https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/bertrand-gabioud-CpuFzIsHYJ0_mask.png\"\n",
+        "init_image = download_image(img_url).resize((512, 512))\n",
+        "mask_image = download_image(mask_url).resize((512, 512))\n",
+        "display(init_image)\n",
+        "display(mask_image)\n",
+        "\n",
+        "instances = [\n",
+        "    {\n",
+        "        \"prompt\": \"a tree, high resolution, in front of high buildings\",\n",
+        "        \"image\": image_to_base64(init_image),\n",
+        "        \"mask_image\": image_to_base64(mask_image),\n",
+        "    },\n",
+        "]\n",
+        "response = endpoint.predict(instances=instances)\n",
+        "images = [base64_to_image(image) for image in response.predictions]\n",
+        "display(images[0])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f12f8d9c2786"
+      },
+      "source": [
+        "### Clean up resources"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "911406c1561e"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete models.\n",
+        "model.delete()\n",
+        "\n",
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_pytorch_stable_diffusion_inpainting.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_tfvision_image_classification.ipynb

@@ -0,0 +1,797 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TirJ-SGQseby"
+      },
+      "source": [
+        "# Vertex AI Model Garden TFVision With Image Classification\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_tfvision_image_classification.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_tfvision_image_classification.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>                                                                                               <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_tfvision_image_classification.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dwGLvtIeECLK"
+      },
+      "source": [
+        "**_NOTE_**: This notebook has been tested in the following environment:\n",
+        "\n",
+        "* Python version = 3.9"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates how to use [TFVision](https://github.com/tensorflow/models/blob/master/official/vision/MODEL_GARDEN.md) in Vertex AI Model Garden.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "* Train new models\n",
+        "  * Convert input data to training formats\n",
+        "  * Create hyperparameter tuning jobs to train new models\n",
+        "  * Find and export best models\n",
+        "\n",
+        "* Test trained models\n",
+        "  * Upload models to model registry\n",
+        "  * Deploy uploaded models\n",
+        "  * Run predictions\n",
+        "\n",
+        "* Cleanup resources\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KEukV6uRk_S3"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "z__i0w0lCAsW"
+      },
+      "source": [
+        "### Colab Only"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Jvqs-ehKlaYh"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install --upgrade google-cloud-aiplatform\n",
+        "\n",
+        "# Automatically restart kernel after installs\n",
+        "import IPython\n",
+        "\n",
+        "app = IPython.Application.instance()\n",
+        "app.kernel.do_shutdown(True)\n",
+        "\n",
+        "from google.colab import auth as google_auth\n",
+        "\n",
+        "google_auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9wExiMUxFk91"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "# The project and bucket are for experiments below.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = \"\"  # @param {type:\"string\"}\n",
+        "REGION = \"us-central1\"\n",
+        "\n",
+        "! gcloud config set project $PROJECT_ID\n",
+        "\n",
+        "STAGING_BUCKET = os.path.join(BUCKET_URI, \"temporal\")\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=STAGING_BUCKET)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "n6IFz75WGCam"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "riG_qUokg0XZ"
+      },
+      "outputs": [],
+      "source": [
+        "OBJECTIVE = \"icn\"\n",
+        "\n",
+        "# Data converter constants.\n",
+        "DATA_CONVERTER_JOB_PREFIX = \"data_converter\"\n",
+        "DATA_CONVERTER_CONTAINER = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/data-converter:latest\"\n",
+        "DATA_CONVERTER_MACHINE_TYPE = \"n1-highmem-8\"\n",
+        "\n",
+        "\n",
+        "# Training constants.\n",
+        "TRAINING_JOB_PREFIX = \"train\"\n",
+        "TRAIN_CONTAINER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/tfvision-oss:latest\"\n",
+        "TRAIN_MACHINE_TYPE = \"n1-highmem-16\"\n",
+        "TRAIN_ACCELERATOR_TYPE = \"NVIDIA_TESLA_P100\"\n",
+        "TRAIN_NUM_GPU = 1\n",
+        "\n",
+        "# Evaluation constants.\n",
+        "EVALUATION_METRIC = \"accuracy\"\n",
+        "\n",
+        "# Export constants.\n",
+        "EXPORT_JOB_PREFIX = \"export\"\n",
+        "EXPORT_CONTAINER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/tfvision-serving:latest\"\n",
+        "EXPORT_MACHINE_TYPE = \"n1-highmem-8\"\n",
+        "\n",
+        "# Prediction constants.\n",
+        "# You can deploy models with\n",
+        "#   pre-build-dockers: https://cloud.google.com/vertex-ai/docs/predictions/pre-built-containers.\n",
+        "#   and optimized tensorflow runtime dockers: https://cloud.google.com/vertex-ai/docs/predictions/optimized-tensorflow-runtime.\n",
+        "# The example in this notebook uses optimized tensorflow runtime dockers.\n",
+        "# You can adjust accelerator types and machine types to get faster predictions.\n",
+        "PREDICTION_CONTAINER_URI = (\n",
+        "    \"us-docker.pkg.dev/vertex-ai-restricted/prediction/tf_opt-gpu.2-11:latest\"\n",
+        ")\n",
+        "SERVING_CONTAINER_ARGS = [\"--allow_precompilation\", \"--allow_compression\"]\n",
+        "PREDICTION_ACCELERATOR_TYPE = \"NVIDIA_TESLA_T4\"\n",
+        "PREDICTION_MACHINE_TYPE = \"n1-standard-4\"\n",
+        "UPLOAD_JOB_PREFIX = \"upload\"\n",
+        "DEPLOY_JOB_PREFIX = \"deploy\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZZFPe_GezXg8"
+      },
+      "source": [
+        "### Define common libraries"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "XcYUGwr-AJGY"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "import json\n",
+        "import os\n",
+        "from datetime import datetime\n",
+        "from io import BytesIO\n",
+        "from typing import Dict, List, Union\n",
+        "\n",
+        "import matplotlib.pyplot as plt\n",
+        "import numpy\n",
+        "import tensorflow as tf\n",
+        "import yaml\n",
+        "from google.cloud import aiplatform\n",
+        "from google.protobuf import json_format\n",
+        "from google.protobuf.struct_pb2 import Value\n",
+        "from PIL import Image\n",
+        "\n",
+        "\n",
+        "def get_job_name_with_datetime(prefix: str):\n",
+        "    return prefix + datetime.now().strftime(\"_%Y%m%d_%H%M%S\")\n",
+        "\n",
+        "\n",
+        "def predict_custom_trained_model(\n",
+        "    project: str,\n",
+        "    endpoint_id: str,\n",
+        "    instances: Union[Dict, List[Dict]],\n",
+        "    location: str = \"us-central1\",\n",
+        "    api_endpoint: str = \"us-central1-aiplatform.googleapis.com\",\n",
+        "):\n",
+        "    # The AI Platform services require regional API endpoints.\n",
+        "    client_options = {\"api_endpoint\": api_endpoint}\n",
+        "    # Initialize client that will be used to create and send requests.\n",
+        "    # This client only needs to be created once, and can be reused for multiple requests.\n",
+        "    client = aiplatform.gapic.PredictionServiceClient(client_options=client_options)\n",
+        "    parameters_dict = {}\n",
+        "    parameters = json_format.ParseDict(parameters_dict, Value())\n",
+        "    endpoint = client.endpoint_path(\n",
+        "        project=project, location=location, endpoint=endpoint_id\n",
+        "    )\n",
+        "    response = client.predict(\n",
+        "        endpoint=endpoint, instances=instances, parameters=parameters\n",
+        "    )\n",
+        "    return response.predictions, response.deployed_model_id\n",
+        "\n",
+        "\n",
+        "def load_img(path):\n",
+        "    img = tf.io.read_file(path)\n",
+        "    img = tf.image.decode_jpeg(img, channels=3)\n",
+        "    return Image.fromarray(numpy.uint8(img)).convert(\"RGB\")\n",
+        "\n",
+        "\n",
+        "def display_image(image):\n",
+        "    _ = plt.figure(figsize=(20, 15))\n",
+        "    plt.grid(False)\n",
+        "    plt.imshow(image)\n",
+        "\n",
+        "\n",
+        "def get_prediction_instances(test_filepath, new_width=-1):\n",
+        "    if new_width <= 0:\n",
+        "        with tf.io.gfile.GFile(test_filepath, \"rb\") as input_file:\n",
+        "            encoded_string = base64.b64encode(input_file.read()).decode(\"utf-8\")\n",
+        "    else:\n",
+        "        img = load_img(test_filepath)\n",
+        "        width, height = img.size\n",
+        "        print(\"original input image size: \", width, \" , \", height)\n",
+        "        new_height = int(height * new_width / width)\n",
+        "        new_img = img.resize((new_width, new_height))\n",
+        "        print(\"resized input image size: \", new_width, \" , \", new_height)\n",
+        "        buffered = BytesIO()\n",
+        "        new_img.save(buffered, format=\"JPEG\")\n",
+        "        encoded_string = base64.b64encode(buffered.getvalue()).decode(\"utf-8\")\n",
+        "\n",
+        "    instances = [\n",
+        "        {\n",
+        "            \"encoded_image\": {\"b64\": encoded_string},\n",
+        "        }\n",
+        "    ]\n",
+        "    return instances\n",
+        "\n",
+        "\n",
+        "def get_label_map(label_map_yaml_filepath):\n",
+        "    with tf.io.gfile.GFile(label_map_yaml_filepath, \"rb\") as input_file:\n",
+        "        label_map = yaml.safe_load(input_file.read())\n",
+        "    return label_map\n",
+        "\n",
+        "\n",
+        "def get_best_trial(model_dir, max_trial_count, evaluation_metric):\n",
+        "    best_trial_dir = \"\"\n",
+        "    best_trial_evaluation_results = {}\n",
+        "    best_performance = -1\n",
+        "\n",
+        "    for i in range(max_trial_count):\n",
+        "        current_trial = i + 1\n",
+        "        current_trial_dir = os.path.join(model_dir, \"trial_\" + str(current_trial))\n",
+        "        current_trial_best_ckpt_dir = os.path.join(current_trial_dir, \"best_ckpt\")\n",
+        "        current_trial_best_ckpt_evaluation_filepath = os.path.join(\n",
+        "            current_trial_best_ckpt_dir, \"info.json\"\n",
+        "        )\n",
+        "        with tf.io.gfile.GFile(current_trial_best_ckpt_evaluation_filepath, \"rb\") as f:\n",
+        "            eval_metric_results = json.load(f)\n",
+        "            current_performance = eval_metric_results[evaluation_metric]\n",
+        "            if current_performance > best_performance:\n",
+        "                best_performance = current_performance\n",
+        "                best_trial_dir = current_trial_dir\n",
+        "                best_trial_evaluation_results = eval_metric_results\n",
+        "    return best_trial_dir, best_trial_evaluation_results"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RB_xY9ipr7ZU"
+      },
+      "source": [
+        "## Train new models\n",
+        "This section will show how to train new models, including:\n",
+        "  * Converting input data to training formats\n",
+        "  * Creating hyperparameter tuning jobs to train new models\n",
+        "  * Finding and export best models\n",
+        "\n",
+        "If you already trained models, please go to the section `Test Trained models`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zgPO1eR3CYjk"
+      },
+      "source": [
+        "### Prepare input data for training\n",
+        "\n",
+        "Prepare data in the format as described [here](https://cloud.google.com/vertex-ai/docs/image-data/classification/prepare-data), and then convert them to the training formats as below:\n",
+        "\n",
+        "* `input_file_path`: The input file path with prepared data.\n",
+        "* `input_file_type`: The input file type, such as csv or jsonl.\n",
+        "* `split_ratio`: The proportion of data to split into train/validation/test.\n",
+        "* `num_shard`: The number of shards for train/validation/test.\n",
+        "* `output_dir`: The output directory, which will container prepared train/test/validation data."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IndQ_m6ddUEM"
+      },
+      "outputs": [],
+      "source": [
+        "# This job will convert input data as training format, with given split ratios\n",
+        "# and number of shards on train/test/validation.\n",
+        "\n",
+        "import os\n",
+        "\n",
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "data_converter_job_name = get_job_name_with_datetime(\n",
+        "    DATA_CONVERTER_JOB_PREFIX + \"_\" + OBJECTIVE\n",
+        ")\n",
+        "\n",
+        "input_file_path = \"\"  # @param {type:\"string\"}\n",
+        "input_file_type = \"\"  # @param {type:\"string\"}\n",
+        "split_ratio = \"0.8,0.1,0.1\"\n",
+        "num_shard = \"10,10,10\"\n",
+        "data_converter_output_dir = os.path.join(BUCKET_URI, data_converter_job_name)\n",
+        "\n",
+        "\n",
+        "worker_pool_specs = [\n",
+        "    {\n",
+        "        \"machine_spec\": {\n",
+        "            \"machine_type\": DATA_CONVERTER_MACHINE_TYPE,\n",
+        "        },\n",
+        "        \"replica_count\": 1,\n",
+        "        \"container_spec\": {\n",
+        "            \"image_uri\": DATA_CONVERTER_CONTAINER,\n",
+        "            \"command\": [],\n",
+        "            \"args\": [\n",
+        "                \"--input_file_path=%s\" % input_file_path,\n",
+        "                \"--input_file_type=%s\" % input_file_type,\n",
+        "                \"--objective=%s\" % OBJECTIVE,\n",
+        "                \"--num_shard=%s\" % num_shard,\n",
+        "                \"--split_ratio=%s\" % split_ratio,\n",
+        "                \"--output_dir=%s\" % data_converter_output_dir,\n",
+        "            ],\n",
+        "        },\n",
+        "    }\n",
+        "]\n",
+        "\n",
+        "data_converter_custom_job = aiplatform.CustomJob(\n",
+        "    display_name=data_converter_job_name,\n",
+        "    project=PROJECT_ID,\n",
+        "    worker_pool_specs=worker_pool_specs,\n",
+        "    staging_bucket=STAGING_BUCKET,\n",
+        ")\n",
+        "\n",
+        "data_converter_custom_job.run()\n",
+        "\n",
+        "input_train_data_path = os.path.join(data_converter_output_dir, \"train.tfrecord*\")\n",
+        "input_validation_data_path = os.path.join(data_converter_output_dir, \"val.tfrecord*\")\n",
+        "label_map_path = os.path.join(data_converter_output_dir, \"label_map.yaml\")\n",
+        "print(\"input_train_data_path for training: \", input_train_data_path)\n",
+        "print(\"input_validation_data_path for training: \", input_validation_data_path)\n",
+        "print(\"label_map_path for prediction: \", label_map_path)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aaff6f5be7f6"
+      },
+      "source": [
+        "### Create a Vertex AI custom job with hyperparameter tuning\n",
+        "\n",
+        "You use the Vertex AI SDK to create and run the hyperparameter tuning job with Vertex AI Model Garden Training Dockers.\n",
+        "\n",
+        "You define the following specifications:\n",
+        "* `worker_pool_specs`: Dictionary specifying the machine type and Docker image. This example defines a single node cluster with one `n1-standard-4` machine with two `NVIDIA_TESLA_T4` GPUs.\n",
+        "* `parameter_spec`: Dictionary specifying the parameters to optimize. The dictionary key is the string assigned to the command line argument for each hyperparameter in your training application code, and the dictionary value is the parameter specification. The parameter specification includes the type, min/max values, and scale for the hyperparameter.\n",
+        "* `metric_spec`: Dictionary specifying the metric to optimize. The dictionary key is the `hyperparameter_metric_tag` that you set in your training application code, and the value is the optimization goal."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "um_XKbmpTaHx"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "from google.cloud import aiplatform\n",
+        "from google.cloud.aiplatform import hyperparameter_tuning as hpt\n",
+        "\n",
+        "# Input train and validation datasets can be found from the section above\n",
+        "# `Convert input data for training`.\n",
+        "# Set prepared datasets if exists.\n",
+        "# input_train_data_path = ''\n",
+        "# input_validation_data_path = ''\n",
+        "\n",
+        "experiment = \"deit_imagenet_pretrain\"  # @param [\"deit_imagenet_pretrain\"]\n",
+        "\n",
+        "# The arguments here are mainly for test purposes. Please update them\n",
+        "# to get better performances.\n",
+        "experiment_container_args_dict = {\n",
+        "    # ViT experiment args.\n",
+        "    \"deit_imagenet_pretrain\": {\n",
+        "        \"experiment\": experiment,\n",
+        "        \"config_file\": \"\",\n",
+        "        \"input_train_data_path\": input_train_data_path,\n",
+        "        \"input_validation_data_path\": input_validation_data_path,\n",
+        "        \"global_batch_size\": 4,\n",
+        "        \"prefetch_buffer_size\": 32,\n",
+        "        \"train_steps\": 500,\n",
+        "        \"input_size\": \"224,224\",\n",
+        "    }\n",
+        "}\n",
+        "\n",
+        "train_job_name = get_job_name_with_datetime(TRAINING_JOB_PREFIX + \"_\" + OBJECTIVE)\n",
+        "model_dir = os.path.join(BUCKET_URI, train_job_name)\n",
+        "\n",
+        "worker_pool_specs = [\n",
+        "    {\n",
+        "        \"machine_spec\": {\n",
+        "            \"machine_type\": TRAIN_MACHINE_TYPE,\n",
+        "            \"accelerator_type\": TRAIN_ACCELERATOR_TYPE,\n",
+        "            # Each training job uses 1 GPU.\n",
+        "            \"accelerator_count\": 1,\n",
+        "        },\n",
+        "        \"replica_count\": 1,\n",
+        "        \"container_spec\": {\n",
+        "            \"image_uri\": TRAIN_CONTAINER_URI,\n",
+        "            \"args\": [\n",
+        "                \"--objective=%s\" % OBJECTIVE,\n",
+        "                \"--mode=train_and_eval\",\n",
+        "                \"--model_dir=%s\" % model_dir,\n",
+        "                \"--params_override=runtime.num_gpus=1\",\n",
+        "            ]\n",
+        "            + [\n",
+        "                \"--{}={}\".format(k, v)\n",
+        "                for k, v in experiment_container_args_dict[experiment].items()\n",
+        "            ],\n",
+        "        },\n",
+        "    }\n",
+        "]\n",
+        "\n",
+        "metric_spec = {\"model_performance\": \"maximize\"}\n",
+        "\n",
+        "\n",
+        "LEARNING_RATES = [5e-4, 1e-3]\n",
+        "# Models will be trained with each learning rate separately and max trial count is the number of learning rates.\n",
+        "MAX_TRIAL_COUNT = len(LEARNING_RATES)\n",
+        "parameter_spec = {\n",
+        "    \"learning_rate\": hpt.DiscreteParameterSpec(values=LEARNING_RATES, scale=\"linear\"),\n",
+        "}\n",
+        "\n",
+        "print(worker_pool_specs, metric_spec, parameter_spec)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "HwcCjwlBTQIz"
+      },
+      "source": [
+        "Then, create and run a `HyperparameterTuningJob`.\n",
+        "* `max_trial_count`: Sets an upper bound on the number of trials the service will run. The recommended practice is to start with a smaller number of trials and get a sense of how impactful your chosen hyperparameters are before scaling up.\n",
+        "\n",
+        "* `parallel_trial_count`:  If you use parallel trials, the service provisions multiple training processing clusters. The worker pool spec that you specify when creating the job is used for each individual training cluster.  Increasing the number of parallel trials reduces the amount of time the hyperparameter tuning job takes to run; however, it can reduce the effectiveness of the job overall. This is because the default tuning strategy uses results of previous trials to inform the assignment of values in subsequent trials.\n",
+        "\n",
+        "* `search_algorithm`: The available search algorithms are grid, random, or default (None). The default option applies Bayesian optimization to search the space of possible hyperparameter values and is the recommended algorithm.\n",
+        "\n",
+        "Click on the generated link in the output to see your run in the Cloud Console."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aec22792ee84"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "train_custom_job = aiplatform.CustomJob(\n",
+        "    display_name=train_job_name,\n",
+        "    project=PROJECT_ID,\n",
+        "    worker_pool_specs=worker_pool_specs,\n",
+        "    staging_bucket=STAGING_BUCKET,\n",
+        ")\n",
+        "\n",
+        "train_hpt_job = aiplatform.HyperparameterTuningJob(\n",
+        "    display_name=train_job_name,\n",
+        "    custom_job=train_custom_job,\n",
+        "    metric_spec=metric_spec,\n",
+        "    parameter_spec=parameter_spec,\n",
+        "    max_trial_count=MAX_TRIAL_COUNT,\n",
+        "    parallel_trial_count=MAX_TRIAL_COUNT,\n",
+        "    project=PROJECT_ID,\n",
+        "    search_algorithm=None,\n",
+        ")\n",
+        "\n",
+        "train_hpt_job.run()\n",
+        "\n",
+        "print(\"experiment is: \", experiment)\n",
+        "print(\"model_dir is: \", model_dir)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mV-Djz-frBni"
+      },
+      "source": [
+        "### Export best models as TF Saved Model format"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "09Rz1AYspK19"
+      },
+      "outputs": [],
+      "source": [
+        "# This job will export models from TF checkpoints to TF saved model format.\n",
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "# model_dir is from the section above.\n",
+        "best_trial_dir, best_trial_evaluation_results = get_best_trial(\n",
+        "    model_dir, MAX_TRIAL_COUNT, EVALUATION_METRIC\n",
+        ")\n",
+        "print(\"best_trial_dir: \", best_trial_dir)\n",
+        "print(\"best_trial_evaluation_results: \", best_trial_evaluation_results)\n",
+        "\n",
+        "worker_pool_specs = [\n",
+        "    {\n",
+        "        \"machine_spec\": {\n",
+        "            \"machine_type\": EXPORT_MACHINE_TYPE,\n",
+        "        },\n",
+        "        \"replica_count\": 1,\n",
+        "        \"container_spec\": {\n",
+        "            \"image_uri\": EXPORT_CONTAINER_URI,\n",
+        "            \"command\": [],\n",
+        "            \"args\": [\n",
+        "                \"--objective=%s\" % OBJECTIVE,\n",
+        "                \"--input_image_size=224,224\",\n",
+        "                \"--experiment=%s\" % experiment,\n",
+        "                \"--config_file=%s/params.yaml\" % best_trial_dir,\n",
+        "                \"--checkpoint_path=%s/best_ckpt\" % best_trial_dir,\n",
+        "                \"--export_dir=%s/best_model\" % model_dir,\n",
+        "            ],\n",
+        "        },\n",
+        "    }\n",
+        "]\n",
+        "\n",
+        "model_export_name = get_job_name_with_datetime(EXPORT_JOB_PREFIX + \"_\" + OBJECTIVE)\n",
+        "model_export_custom_job = aiplatform.CustomJob(\n",
+        "    display_name=model_export_name,\n",
+        "    project=PROJECT_ID,\n",
+        "    worker_pool_specs=worker_pool_specs,\n",
+        "    staging_bucket=STAGING_BUCKET,\n",
+        ")\n",
+        "\n",
+        "\n",
+        "model_export_custom_job.run()\n",
+        "\n",
+        "print(\"best model is saved to: \", os.path.join(model_dir, \"best_model\"))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "g0BGaofgsMsy"
+      },
+      "source": [
+        "## Test trained models\n",
+        "This section will show how to tests with trained models, including:\n",
+        "  * uploading models to model registry\n",
+        "  * deploying uploaded models\n",
+        "  * running predictions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NYuQowyZEtxK"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Upload and deploy models\n",
+        "import os\n",
+        "\n",
+        "# model_dir is from the section above.\n",
+        "trained_model_dir = os.path.join(model_dir, \"best_model/saved_model\")\n",
+        "\n",
+        "upload_job_name = get_job_name_with_datetime(UPLOAD_JOB_PREFIX + \"_\" + OBJECTIVE)\n",
+        "\n",
+        "model = aiplatform.Model.upload(\n",
+        "    display_name=upload_job_name,\n",
+        "    artifact_uri=trained_model_dir,\n",
+        "    serving_container_image_uri=PREDICTION_CONTAINER_URI,\n",
+        "    serving_container_args=SERVING_CONTAINER_ARGS,\n",
+        ")\n",
+        "\n",
+        "model.wait()\n",
+        "\n",
+        "print(\"The uploaded model name is: \", upload_job_name)\n",
+        "\n",
+        "deploy_model_name = get_job_name_with_datetime(DEPLOY_JOB_PREFIX + \"_\" + OBJECTIVE)\n",
+        "print(\"The deployed job name is: \", deploy_model_name)\n",
+        "\n",
+        "endpoint = model.deploy(\n",
+        "    deployed_model_display_name=deploy_model_name,\n",
+        "    machine_type=PREDICTION_MACHINE_TYPE,\n",
+        "    traffic_split={\"0\": 100},\n",
+        "    accelerator_type=PREDICTION_ACCELERATOR_TYPE,\n",
+        "    accelerator_count=1,\n",
+        "    min_replica_count=1,\n",
+        "    max_replica_count=1,\n",
+        ")\n",
+        "\n",
+        "endpoint_id = endpoint.name\n",
+        "print(\"endpoint id is: \", endpoint_id)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vbIW9me1F2RY"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Run predictions\n",
+        "\n",
+        "# endpoint_id was generated in the section above (`Upload and deploy models`).\n",
+        "endpoint_id = endpoint.name\n",
+        "\n",
+        "test_filepath = \"\"  # @param {type:\"string\"}\n",
+        "# If the input image is too large, we will resize it for prediction.\n",
+        "instances = get_prediction_instances(test_filepath, new_width=1000)\n",
+        "\n",
+        "# The label map file was generated from the section above (`Convert input data for training`).\n",
+        "label_map = get_label_map(label_map_path)[\"label_map\"]\n",
+        "\n",
+        "predictions, _ = predict_custom_trained_model(\n",
+        "    project=PROJECT_ID, location=REGION, endpoint_id=endpoint_id, instances=instances\n",
+        ")\n",
+        "\n",
+        "probs = dict(predictions[0])[\"probs\"]\n",
+        "max_prob = max(probs)\n",
+        "max_index = probs.index(max_prob)\n",
+        "print(\"The test image: \", test_filepath)\n",
+        "print(\"max_prob: \", max_prob, \", for label: \", label_map[max_index])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kkH2nrpdp4sp"
+      },
+      "source": [
+        "## Clean up"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ax6vQVZhp9pR"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete models.\n",
+        "model.delete()\n",
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_tfvision_image_classification.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/model_garden/model_garden_tfvision_image_object_detection.ipynb

@@ -0,0 +1,961 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TirJ-SGQseby"
+      },
+      "source": [
+        "# Vertex AI Model Garden TFVision With Image Object Detection\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_tfvision_image_object_detection.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_tfvision_image_object_detection.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>                                                                                               <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_tfvision_image_object_detection.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dwGLvtIeECLK"
+      },
+      "source": [
+        "**_NOTE_**: This notebook has been tested in the following environment:\n",
+        "\n",
+        "* Python version = 3.9"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates how to use [TFVision](https://github.com/tensorflow/models/blob/master/official/vision/MODEL_GARDEN.md) in Vertex AI Model Garden.\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "* Train new models\n",
+        "  * Convert input data to training formats\n",
+        "  * Create [hyperparameter tuning jobs](https://cloud.google.com/vertex-ai/docs/training/hyperparameter-tuning-overview) to train new models\n",
+        "  * Find and export best models\n",
+        "\n",
+        "* Test trained models\n",
+        "  * Upload models to model registry\n",
+        "  * Deploy uploaded models\n",
+        "  * Run predictions\n",
+        "\n",
+        "* Cleanup resources\n",
+        "\n",
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KEukV6uRk_S3"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "z__i0w0lCAsW"
+      },
+      "source": [
+        "### Colab Only\n",
+        "Run the following cmds for colab and skip this section if you use workbench."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Jvqs-ehKlaYh"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install --upgrade google-cloud-aiplatform\n",
+        "\n",
+        "# Automatically restart kernel after installs\n",
+        "import IPython\n",
+        "\n",
+        "app = IPython.Application.instance()\n",
+        "app.kernel.do_shutdown(True)\n",
+        "\n",
+        "from google.colab import auth as google_auth\n",
+        "\n",
+        "google_auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9wExiMUxFk91"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "# The project and bucket are for experiments below.\n",
+        "PROJECT_ID = \"\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = \"\"  # @param {type:\"string\"}\n",
+        "REGION = \"us-central1\"\n",
+        "\n",
+        "! gcloud config set project $PROJECT_ID\n",
+        "\n",
+        "STAGING_BUCKET = os.path.join(BUCKET_URI, \"temporal\")\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=STAGING_BUCKET)\n",
+        "\n",
+        "# Download config files.\n",
+        "CONFIG_DIR = os.path.join(BUCKET_URI, \"config\")\n",
+        "! wget https://raw.githubusercontent.com/tensorflow/models/master/official/vision/configs/experiments/retinanet/coco_spinenet49_gpu_multiworker_mirrored.yaml\n",
+        "! gsutil cp coco_spinenet49_gpu_multiworker_mirrored.yaml $CONFIG_DIR\n",
+        "\n",
+        "! wget https://raw.githubusercontent.com/tensorflow/models/master/official/vision/configs/experiments/retinanet/coco_spinenet96_gpu_multiworker_mirrored.yaml\n",
+        "! gsutil cp coco_spinenet96_gpu_multiworker_mirrored.yaml $CONFIG_DIR\n",
+        "\n",
+        "! wget https://raw.githubusercontent.com/tensorflow/models/master/official/vision/configs/experiments/retinanet/coco_spinenet143_gpu_multiworker_mirrored.yaml\n",
+        "! gsutil cp coco_spinenet143_gpu_multiworker_mirrored.yaml $CONFIG_DIR\n",
+        "\n",
+        "! wget https://raw.githubusercontent.com/tensorflow/models/master/official/projects/yolo/configs/experiments/yolov4/detection/scaled_yolov4_1280_gpu.yaml\n",
+        "! gsutil cp scaled_yolov4_1280_gpu.yaml $CONFIG_DIR"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "n6IFz75WGCam"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "riG_qUokg0XZ"
+      },
+      "outputs": [],
+      "source": [
+        "OBJECTIVE = \"iod\"\n",
+        "\n",
+        "# Data converter constants.\n",
+        "DATA_CONVERTER_JOB_PREFIX = \"data_converter\"\n",
+        "DATA_CONVERTER_CONTAINER = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/data-converter:latest\"\n",
+        "DATA_CONVERTER_MACHINE_TYPE = \"n1-highmem-8\"\n",
+        "\n",
+        "\n",
+        "# Training constants.\n",
+        "TRAINING_JOB_PREFIX = \"train\"\n",
+        "TRAIN_CONTAINER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/tfvision-oss:latest\"\n",
+        "TRAIN_MACHINE_TYPE = \"n1-highmem-16\"\n",
+        "TRAIN_ACCELERATOR_TYPE = \"NVIDIA_TESLA_V100\"\n",
+        "TRAIN_NUM_GPU = 2\n",
+        "TRAIN_SPINENET49_CONFIG = os.path.join(\n",
+        "    CONFIG_DIR, \"coco_spinenet49_gpu_multiworker_mirrored.yaml\"\n",
+        ")\n",
+        "TRAIN_SPINENET96_CONFIG = os.path.join(\n",
+        "    CONFIG_DIR, \"coco_spinenet96_gpu_multiworker_mirrored.yaml\"\n",
+        ")\n",
+        "TRAIN_SPINENET143_CONFIG = os.path.join(\n",
+        "    CONFIG_DIR, \"coco_spinenet143_gpu_multiworker_mirrored.yaml\"\n",
+        ")\n",
+        "TRAIN_YOLOV4_CONFIG = os.path.join(CONFIG_DIR, \"scaled_yolov4_1280_gpu.yaml\")\n",
+        "\n",
+        "# Evaluation constants.\n",
+        "EVALUATION_METRIC = \"AP50\"\n",
+        "\n",
+        "# Export constants.\n",
+        "EXPORT_JOB_PREFIX = \"export\"\n",
+        "EXPORT_CONTAINER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/tfvision-serving:latest\"\n",
+        "EXPORT_MACHINE_TYPE = \"n1-highmem-8\"\n",
+        "\n",
+        "# Prediction constants.\n",
+        "# You can deploy models with\n",
+        "#   pre-build-dockers: https://cloud.google.com/vertex-ai/docs/predictions/pre-built-containers.\n",
+        "#   and optimized tensorflow runtime dockers: https://cloud.google.com/vertex-ai/docs/predictions/optimized-tensorflow-runtime.\n",
+        "# The example in this notebook uses optimized tensorflow runtime dockers.\n",
+        "# You can adjust accelerator types and machine types to get faster predictions.\n",
+        "PREDICTION_CONTAINER_URI = (\n",
+        "    \"us-docker.pkg.dev/vertex-ai-restricted/prediction/tf_opt-gpu.2-11:latest\"\n",
+        ")\n",
+        "SERVING_CONTAINER_ARGS = [\"--allow_precompilation\", \"--allow_compression\"]\n",
+        "PREDICTION_ACCELERATOR_TYPE = \"NVIDIA_TESLA_T4\"\n",
+        "PREDICTION_MACHINE_TYPE = \"n1-standard-4\"\n",
+        "UPLOAD_JOB_PREFIX = \"upload\"\n",
+        "DEPLOY_JOB_PREFIX = \"deploy\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZZFPe_GezXg8"
+      },
+      "source": [
+        "### Define common libraries"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "XcYUGwr-AJGY"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "import json\n",
+        "import os\n",
+        "from datetime import datetime\n",
+        "from io import BytesIO\n",
+        "from typing import Dict, List, Union\n",
+        "\n",
+        "import matplotlib.pyplot as plt\n",
+        "import numpy as np\n",
+        "import tensorflow as tf\n",
+        "import yaml\n",
+        "from google.cloud import aiplatform\n",
+        "from google.protobuf import json_format\n",
+        "from google.protobuf.struct_pb2 import Value\n",
+        "from PIL import Image, ImageColor, ImageDraw, ImageFont\n",
+        "\n",
+        "\n",
+        "def get_job_name_with_datetime(prefix: str):\n",
+        "    return prefix + datetime.now().strftime(\"_%Y%m%d_%H%M%S\")\n",
+        "\n",
+        "\n",
+        "def predict_custom_trained_model(\n",
+        "    project: str,\n",
+        "    endpoint_id: str,\n",
+        "    instances: Union[Dict, List[Dict]],\n",
+        "    location: str = \"us-central1\",\n",
+        "    api_endpoint: str = \"us-central1-aiplatform.googleapis.com\",\n",
+        "):\n",
+        "    # The AI Platform services require regional API endpoints.\n",
+        "    client_options = {\"api_endpoint\": api_endpoint}\n",
+        "    # Initialize client that will be used to create and send requests.\n",
+        "    # This client only needs to be created once, and can be reused for multiple requests.\n",
+        "    client = aiplatform.gapic.PredictionServiceClient(client_options=client_options)\n",
+        "    parameters_dict = {}\n",
+        "    parameters = json_format.ParseDict(parameters_dict, Value())\n",
+        "    endpoint = client.endpoint_path(\n",
+        "        project=project, location=location, endpoint=endpoint_id\n",
+        "    )\n",
+        "    response = client.predict(\n",
+        "        endpoint=endpoint, instances=instances, parameters=parameters\n",
+        "    )\n",
+        "    return response.predictions, response.deployed_model_id\n",
+        "\n",
+        "\n",
+        "def load_img(path):\n",
+        "    img = tf.io.read_file(path)\n",
+        "    img = tf.image.decode_jpeg(img, channels=3)\n",
+        "    return Image.fromarray(np.uint8(img)).convert(\"RGB\")\n",
+        "\n",
+        "\n",
+        "def display_image(image):\n",
+        "    _ = plt.figure(figsize=(20, 15))\n",
+        "    plt.grid(False)\n",
+        "    plt.imshow(image)\n",
+        "\n",
+        "\n",
+        "def get_prediction_instances(test_filepath, new_width=-1):\n",
+        "    if new_width <= 0:\n",
+        "        with tf.io.gfile.GFile(test_filepath, \"rb\") as input_file:\n",
+        "            encoded_string = base64.b64encode(input_file.read()).decode(\"utf-8\")\n",
+        "    else:\n",
+        "        img = load_img(test_filepath)\n",
+        "        width, height = img.size\n",
+        "        print(\"original input image size: \", width, \" , \", height)\n",
+        "        new_height = int(height * new_width / width)\n",
+        "        new_img = img.resize((new_width, new_height))\n",
+        "        print(\"resized input image size: \", new_width, \" , \", new_height)\n",
+        "        buffered = BytesIO()\n",
+        "        new_img.save(buffered, format=\"JPEG\")\n",
+        "        encoded_string = base64.b64encode(buffered.getvalue()).decode(\"utf-8\")\n",
+        "\n",
+        "    instances = [\n",
+        "        {\n",
+        "            \"encoded_image\": {\"b64\": encoded_string},\n",
+        "        }\n",
+        "    ]\n",
+        "    return instances\n",
+        "\n",
+        "\n",
+        "def get_label_map(label_map_yaml_filepath):\n",
+        "    with tf.io.gfile.GFile(label_map_yaml_filepath, \"rb\") as input_file:\n",
+        "        label_map = yaml.safe_load(input_file.read())\n",
+        "    return label_map\n",
+        "\n",
+        "\n",
+        "def get_best_trial(model_dir, max_trial_count, evaluation_metric):\n",
+        "    best_trial_dir = \"\"\n",
+        "    best_trial_evaluation_results = {}\n",
+        "    best_performance = -1\n",
+        "\n",
+        "    for i in range(max_trial_count):\n",
+        "        current_trial = i + 1\n",
+        "        current_trial_dir = os.path.join(model_dir, \"trial_\" + str(current_trial))\n",
+        "        current_trial_best_ckpt_dir = os.path.join(current_trial_dir, \"best_ckpt\")\n",
+        "        current_trial_best_ckpt_evaluation_filepath = os.path.join(\n",
+        "            current_trial_best_ckpt_dir, \"info.json\"\n",
+        "        )\n",
+        "        with tf.io.gfile.GFile(current_trial_best_ckpt_evaluation_filepath, \"rb\") as f:\n",
+        "            eval_metric_results = json.load(f)\n",
+        "            current_performance = eval_metric_results[evaluation_metric]\n",
+        "            if current_performance > best_performance:\n",
+        "                best_performance = current_performance\n",
+        "                best_trial_dir = current_trial_dir\n",
+        "                best_trial_evaluation_results = eval_metric_results\n",
+        "    return best_trial_dir, best_trial_evaluation_results\n",
+        "\n",
+        "\n",
+        "def draw_bounding_box_on_image(\n",
+        "    image, ymin, xmin, ymax, xmax, color, font, thickness=4, display_str_list=()\n",
+        "):\n",
+        "    \"\"\"Adds a bounding box to an image.\"\"\"\n",
+        "    draw = ImageDraw.Draw(image)\n",
+        "    im_width, im_height = image.size\n",
+        "    (left, right, top, bottom) = (\n",
+        "        xmin * im_width,\n",
+        "        xmax * im_width,\n",
+        "        ymin * im_height,\n",
+        "        ymax * im_height,\n",
+        "    )\n",
+        "    draw.line(\n",
+        "        [(left, top), (left, bottom), (right, bottom), (right, top), (left, top)],\n",
+        "        width=thickness,\n",
+        "        fill=color,\n",
+        "    )\n",
+        "\n",
+        "    # If the total height of the display strings added to the top of the bounding\n",
+        "    # box exceeds the top of the image, stack the strings below the bounding box\n",
+        "    # instead of above.\n",
+        "    display_str_heights = [font.getsize(ds)[1] for ds in display_str_list]\n",
+        "    # Each display_str has a top and bottom margin of 0.05x.\n",
+        "    total_display_str_height = (1 + 2 * 0.05) * sum(display_str_heights)\n",
+        "\n",
+        "    if top > total_display_str_height:\n",
+        "        text_bottom = top\n",
+        "    else:\n",
+        "        text_bottom = top + total_display_str_height\n",
+        "    # Reverse list and print from bottom to top.\n",
+        "    for display_str in display_str_list[::-1]:\n",
+        "        text_width, text_height = font.getsize(display_str)\n",
+        "        margin = np.ceil(0.05 * text_height)\n",
+        "        draw.rectangle(\n",
+        "            [\n",
+        "                (left, text_bottom - text_height - 2 * margin),\n",
+        "                (left + text_width, text_bottom),\n",
+        "            ],\n",
+        "            fill=color,\n",
+        "        )\n",
+        "        draw.text(\n",
+        "            (left + margin, text_bottom - text_height - margin),\n",
+        "            display_str,\n",
+        "            fill=\"black\",\n",
+        "            font=font,\n",
+        "        )\n",
+        "        text_bottom -= text_height - 2 * margin\n",
+        "\n",
+        "\n",
+        "def draw_boxes(image, boxes, class_names, scores, max_boxes=40, min_score=0.05):\n",
+        "    \"\"\"Overlay labeled boxes on an image with formatted scores and label names.\"\"\"\n",
+        "    colors = list(ImageColor.colormap.values())\n",
+        "    try:\n",
+        "        font = ImageFont.truetype(\n",
+        "            \"/usr/share/fonts/truetype/liberation/LiberationSansNarrow-Regular.ttf\", 25\n",
+        "        )\n",
+        "    except IOError:\n",
+        "        print(\"Font not found, using default font.\")\n",
+        "        font = ImageFont.load_default()\n",
+        "\n",
+        "    for i in range(min(len(boxes), max_boxes)):\n",
+        "        if scores[i] >= min_score:\n",
+        "            ymin, xmin, ymax, xmax = boxes[i]\n",
+        "            display_str = \"{}: {}%\".format(class_names[i], int(100 * scores[i]))\n",
+        "            color = colors[hash(class_names[i]) % len(colors)]\n",
+        "            draw_bounding_box_on_image(\n",
+        "                image,\n",
+        "                ymin,\n",
+        "                xmin,\n",
+        "                ymax,\n",
+        "                xmax,\n",
+        "                color,\n",
+        "                font,\n",
+        "                display_str_list=[display_str],\n",
+        "            )\n",
+        "    return image"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RB_xY9ipr7ZU"
+      },
+      "source": [
+        "## Train new models\n",
+        "This section will show how to train new models.\n",
+        "\n",
+        "  * 1. Convert input data to training formats\n",
+        "  * 2. Create hyperparameter tuning jobs to train new models\n",
+        "  * 3. Find and export best models\n",
+        "\n",
+        "If you already trained models, please go to the section `Test trained models`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zgPO1eR3CYjk"
+      },
+      "source": [
+        "### Prepare input data for training\n",
+        "\n",
+        "Prepare data in the format as described [here](https://cloud.google.com/vertex-ai/docs/image-data/classification/prepare-data), and then convert them to the training formats as below:\n",
+        "\n",
+        "* `input_file_path`: The input file path for preparing data.\n",
+        "* `input_file_type`: The input file type, such as csv or jsonl.\n",
+        "* `split_ratio`: The proportion of data to split into train/validation/test.\n",
+        "* `num_shard`: The number of shards for train/validation/test.\n",
+        "* `output_dir`: The output directory, which will container prepared train/test/validation data."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IndQ_m6ddUEM"
+      },
+      "outputs": [],
+      "source": [
+        "# This job will convert input data as training format, with given split ratios\n",
+        "# and number of shards on train/test/validation.\n",
+        "data_converter_job_name = get_job_name_with_datetime(\n",
+        "    DATA_CONVERTER_JOB_PREFIX + \"_\" + OBJECTIVE\n",
+        ")\n",
+        "\n",
+        "input_file_path = \"\"  # @param {type:\"string\"}\n",
+        "input_file_type = \"\"  # @param {type:\"string\"}\n",
+        "split_ratio = \"0.8,0.1,0.1\"\n",
+        "num_shard = \"10,10,10\"\n",
+        "data_converter_output_dir = os.path.join(BUCKET_URI, data_converter_job_name)\n",
+        "\n",
+        "\n",
+        "worker_pool_specs = [\n",
+        "    {\n",
+        "        \"machine_spec\": {\n",
+        "            \"machine_type\": DATA_CONVERTER_MACHINE_TYPE,\n",
+        "        },\n",
+        "        \"replica_count\": 1,\n",
+        "        \"container_spec\": {\n",
+        "            \"image_uri\": DATA_CONVERTER_CONTAINER,\n",
+        "            \"command\": [],\n",
+        "            \"args\": [\n",
+        "                \"--input_file_path=%s\" % input_file_path,\n",
+        "                \"--input_file_type=%s\" % input_file_type,\n",
+        "                \"--objective=%s\" % OBJECTIVE,\n",
+        "                \"--num_shard=%s\" % num_shard,\n",
+        "                \"--split_ratio=%s\" % split_ratio,\n",
+        "                \"--output_dir=%s\" % data_converter_output_dir,\n",
+        "            ],\n",
+        "        },\n",
+        "    }\n",
+        "]\n",
+        "\n",
+        "data_converter_custom_job = aiplatform.CustomJob(\n",
+        "    display_name=data_converter_job_name,\n",
+        "    project=PROJECT_ID,\n",
+        "    worker_pool_specs=worker_pool_specs,\n",
+        "    staging_bucket=STAGING_BUCKET,\n",
+        ")\n",
+        "\n",
+        "data_converter_custom_job.run()\n",
+        "\n",
+        "input_train_data_path = os.path.join(data_converter_output_dir, \"train.tfrecord*\")\n",
+        "input_validation_data_path = os.path.join(data_converter_output_dir, \"val.tfrecord*\")\n",
+        "label_map_path = os.path.join(data_converter_output_dir, \"label_map.yaml\")\n",
+        "print(\"input_train_data_path for training: \", input_train_data_path)\n",
+        "print(\"input_validation_data_path for training: \", input_validation_data_path)\n",
+        "print(\"label_map_path for prediction: \", label_map_path)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SA8DVTn7j69v"
+      },
+      "source": [
+        "### Create a Vertex AI custom job with hyperparameter tuning\n",
+        "\n",
+        "You use the Vertex AI SDK to create and run the hyperparameter tuning job with Vertex AI Model Garden Training Dockers."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aaff6f5be7f6"
+      },
+      "source": [
+        "#### Define the following specifications\n",
+        "\n",
+        "* `worker_pool_specs`: Dictionary specifying the machine type and Docker image. This example defines a single node cluster with one `n1-standard-4` machine with two `NVIDIA_TESLA_T4` GPUs.\n",
+        "* `parameter_spec`: Dictionary specifying the parameters to optimize. The dictionary key is the string assigned to the command line argument for each hyperparameter in your training application code, and the dictionary value is the parameter specification. The parameter specification includes the type, min/max values, and scale for the hyperparameter.\n",
+        "* `metric_spec`: Dictionary specifying the metric to optimize. The dictionary key is the `hyperparameter_metric_tag` that you set in your training application code, and the value is the optimization goal."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "um_XKbmpTaHx"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud.aiplatform import hyperparameter_tuning as hpt\n",
+        "\n",
+        "label_map = get_label_map(label_map_path)\n",
+        "num_classes = len(label_map[\"label_map\"]) + 1\n",
+        "\n",
+        "# Input train and validation datasets can be found from the section above\n",
+        "# `Convert input data for training`.\n",
+        "# Set prepared datasets if exists.\n",
+        "# input_train_data_path = ''\n",
+        "# input_validation_data_path = ''\n",
+        "\n",
+        "# Refer to https://github.com/tensorflow/models/blob/master/official/vision/MODEL_GARDEN.md\n",
+        "# for more model details.\n",
+        "experiment = \"retinanet_spinenet96\"  # @param ['retinanet_spinenet49', \"retinanet_spinenet96\", 'retinanet_spinenet143', 'scaled_yolo_v4']\n",
+        "\n",
+        "train_job_name = get_job_name_with_datetime(TRAINING_JOB_PREFIX + \"_\" + OBJECTIVE)\n",
+        "model_dir = os.path.join(BUCKET_URI, train_job_name)\n",
+        "\n",
+        "# The arguments here are mainly for test purposes. Please update them\n",
+        "# to get better performances.\n",
+        "common_args = {\n",
+        "    \"input_train_data_path\": input_train_data_path,\n",
+        "    \"input_validation_data_path\": input_validation_data_path,\n",
+        "    \"objective\": OBJECTIVE,\n",
+        "    \"model_dir\": model_dir,\n",
+        "    \"num_classes\": num_classes,\n",
+        "    \"global_batch_size\": 4,\n",
+        "    \"prefetch_buffer_size\": 12,\n",
+        "    \"train_steps\": 2000,\n",
+        "    \"input_size\": \"1024,1024\",\n",
+        "}\n",
+        "\n",
+        "experiment_container_args_dict = {\n",
+        "    # retinanet_spinenet49 experiment args.\n",
+        "    \"retinanet_spinenet49\": {\n",
+        "        \"experiment\": \"retinanet_spinenet_coco\",\n",
+        "        \"config_file\": TRAIN_SPINENET49_CONFIG,\n",
+        "    }\n",
+        "    | common_args,\n",
+        "    # retinanet_spinenet96 experiment args.\n",
+        "    \"retinanet_spinenet96\": {\n",
+        "        \"experiment\": \"retinanet_spinenet_coco\",\n",
+        "        \"config_file\": TRAIN_SPINENET96_CONFIG,\n",
+        "    }\n",
+        "    | common_args,\n",
+        "    # retinanet_spinenet143 experiment args.\n",
+        "    \"retinanet_spinenet143\": {\n",
+        "        \"experiment\": \"retinanet_spinenet_coco\",\n",
+        "        \"config_file\": TRAIN_SPINENET96_CONFIG,\n",
+        "    }\n",
+        "    | common_args,\n",
+        "    # scaled_yolo_v4 experiment args.\n",
+        "    \"scaled_yolo_v4\": {\n",
+        "        \"experiment\": \"scaled_yolo\",\n",
+        "        \"config_file\": TRAIN_YOLOV4_CONFIG,\n",
+        "    }\n",
+        "    | common_args,\n",
+        "}\n",
+        "\n",
+        "params_override = \"runtime.num_gpus=%s\" % TRAIN_NUM_GPU\n",
+        "eval_params_override = \"runtime.num_gpus=1,runtime.distribution_strategy=mirrored\"\n",
+        "worker_pool_specs = [\n",
+        "    {\n",
+        "        \"machine_spec\": {\n",
+        "            \"machine_type\": TRAIN_MACHINE_TYPE,\n",
+        "            \"accelerator_type\": TRAIN_ACCELERATOR_TYPE,\n",
+        "            \"accelerator_count\": TRAIN_NUM_GPU,\n",
+        "        },\n",
+        "        \"replica_count\": 1,\n",
+        "        \"container_spec\": {\n",
+        "            \"image_uri\": TRAIN_CONTAINER_URI,\n",
+        "            \"args\": [\n",
+        "                \"--mode=train\",\n",
+        "                \"--params_override=%s\" % params_override,\n",
+        "            ]\n",
+        "            + [\n",
+        "                \"--{}={}\".format(k, v)\n",
+        "                for k, v in experiment_container_args_dict[experiment].items()\n",
+        "            ],\n",
+        "        },\n",
+        "    },\n",
+        "    {},\n",
+        "    {},\n",
+        "    {\n",
+        "        \"machine_spec\": {\n",
+        "            \"machine_type\": \"n1-highmem-4\",\n",
+        "            \"accelerator_type\": TRAIN_ACCELERATOR_TYPE,\n",
+        "            \"accelerator_count\": 1,\n",
+        "        },\n",
+        "        \"replica_count\": 1,\n",
+        "        \"container_spec\": {\n",
+        "            \"image_uri\": TRAIN_CONTAINER_URI,\n",
+        "            \"args\": [\n",
+        "                \"--mode=continuous_eval\",\n",
+        "                \"--params_override=%s\" % eval_params_override,\n",
+        "            ]\n",
+        "            + [\n",
+        "                \"--{}={}\".format(k, v)\n",
+        "                for k, v in experiment_container_args_dict[experiment].items()\n",
+        "            ],\n",
+        "        },\n",
+        "    },\n",
+        "]\n",
+        "\n",
+        "metric_spec = {\"model_performance\": \"maximize\"}\n",
+        "\n",
+        "LEARNING_RATES = [0.001, 0.01]\n",
+        "# Models will be trained with each learning rate separately and max trial count is the number of learning rates.\n",
+        "MAX_TRIAL_COUNT = len(LEARNING_RATES)\n",
+        "parameter_spec = {\n",
+        "    \"learning_rate\": hpt.DiscreteParameterSpec(values=LEARNING_RATES, scale=\"linear\"),\n",
+        "}\n",
+        "\n",
+        "print(worker_pool_specs, metric_spec, parameter_spec)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "HwcCjwlBTQIz"
+      },
+      "source": [
+        "#### Run hyperparameter tuning jobs\n",
+        "* `max_trial_count`: Sets an upper bound on the number of trials the service will run. The recommended practice is to start with a smaller number of trials and get a sense of how impactful your chosen hyperparameters are before scaling up.\n",
+        "\n",
+        "* `parallel_trial_count`:  If you use parallel trials, the service provisions multiple training processing clusters. The worker pool spec that you specify when creating the job is used for each individual training cluster.  Increasing the number of parallel trials reduces the amount of time the hyperparameter tuning job takes to run; however, it can reduce the effectiveness of the job overall. This is because the default tuning strategy uses results of previous trials to inform the assignment of values in subsequent trials.\n",
+        "\n",
+        "* `search_algorithm`: The available search algorithms are grid, random, or default (None). The default option applies Bayesian optimization to search the space of possible hyperparameter values and is the recommended algorithm.\n",
+        "\n",
+        "Click on the generated link in the output to see your run in the Cloud Console."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aec22792ee84"
+      },
+      "outputs": [],
+      "source": [
+        "train_custom_job = aiplatform.CustomJob(\n",
+        "    display_name=train_job_name,\n",
+        "    project=PROJECT_ID,\n",
+        "    worker_pool_specs=worker_pool_specs,\n",
+        "    staging_bucket=STAGING_BUCKET,\n",
+        ")\n",
+        "\n",
+        "train_hpt_job = aiplatform.HyperparameterTuningJob(\n",
+        "    display_name=train_job_name,\n",
+        "    custom_job=train_custom_job,\n",
+        "    metric_spec=metric_spec,\n",
+        "    parameter_spec=parameter_spec,\n",
+        "    max_trial_count=MAX_TRIAL_COUNT,\n",
+        "    parallel_trial_count=1,\n",
+        "    project=PROJECT_ID,\n",
+        "    search_algorithm=None,\n",
+        ")\n",
+        "\n",
+        "train_hpt_job.run()\n",
+        "\n",
+        "print(\"experiment is: \", experiment)\n",
+        "print(\"model_dir is: \", model_dir)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mV-Djz-frBni"
+      },
+      "source": [
+        "### Export best models as TF Saved Model format"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "09Rz1AYspK19"
+      },
+      "outputs": [],
+      "source": [
+        "# This job will export models from TF checkpoints to TF saved model format.\n",
+        "# model_dir is from the section above.\n",
+        "best_trial_dir, best_trial_evaluation_results = get_best_trial(\n",
+        "    model_dir, MAX_TRIAL_COUNT, EVALUATION_METRIC\n",
+        ")\n",
+        "print(\"best_trial_dir: \", best_trial_dir)\n",
+        "print(\"best_trial_evaluation_results: \", best_trial_evaluation_results)\n",
+        "\n",
+        "worker_pool_specs = [\n",
+        "    {\n",
+        "        \"machine_spec\": {\n",
+        "            \"machine_type\": EXPORT_MACHINE_TYPE,\n",
+        "        },\n",
+        "        \"replica_count\": 1,\n",
+        "        \"container_spec\": {\n",
+        "            \"image_uri\": EXPORT_CONTAINER_URI,\n",
+        "            \"command\": [],\n",
+        "            \"args\": [\n",
+        "                \"--objective=%s\" % OBJECTIVE,\n",
+        "                \"--input_image_size=1024,1024\",\n",
+        "                \"--experiment=%s\"\n",
+        "                % experiment_container_args_dict[experiment][\"experiment\"],\n",
+        "                \"--config_file=%s/params.yaml\" % best_trial_dir,\n",
+        "                \"--checkpoint_path=%s/best_ckpt\" % best_trial_dir,\n",
+        "                \"--export_dir=%s/best_model\" % model_dir,\n",
+        "            ],\n",
+        "        },\n",
+        "    }\n",
+        "]\n",
+        "\n",
+        "model_export_name = get_job_name_with_datetime(EXPORT_JOB_PREFIX + \"_\" + OBJECTIVE)\n",
+        "model_export_custom_job = aiplatform.CustomJob(\n",
+        "    display_name=model_export_name,\n",
+        "    project=PROJECT_ID,\n",
+        "    worker_pool_specs=worker_pool_specs,\n",
+        "    staging_bucket=STAGING_BUCKET,\n",
+        ")\n",
+        "\n",
+        "\n",
+        "model_export_custom_job.run()\n",
+        "\n",
+        "print(\"best model is saved to: \", os.path.join(model_dir, \"best_model\"))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "g0BGaofgsMsy"
+      },
+      "source": [
+        "## Test trained models\n",
+        "This section will show how to test with trained models.\n",
+        "  * 1. Upload and deploy models\n",
+        "  * 2. Run predictions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NYuQowyZEtxK"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Upload and deploy models\n",
+        "# model_dir is from the section above.\n",
+        "trained_model_dir = os.path.join(model_dir, \"best_model/saved_model\")\n",
+        "\n",
+        "upload_job_name = get_job_name_with_datetime(UPLOAD_JOB_PREFIX + \"_\" + OBJECTIVE)\n",
+        "\n",
+        "model = aiplatform.Model.upload(\n",
+        "    display_name=upload_job_name,\n",
+        "    artifact_uri=trained_model_dir,\n",
+        "    serving_container_image_uri=PREDICTION_CONTAINER_URI,\n",
+        "    serving_container_args=SERVING_CONTAINER_ARGS,\n",
+        ")\n",
+        "\n",
+        "model.wait()\n",
+        "\n",
+        "print(\"The uploaded model name is: \", upload_job_name)\n",
+        "\n",
+        "deploy_model_name = get_job_name_with_datetime(DEPLOY_JOB_PREFIX + \"_\" + OBJECTIVE)\n",
+        "print(\"The deployed job name is: \", deploy_model_name)\n",
+        "\n",
+        "endpoint = model.deploy(\n",
+        "    deployed_model_display_name=deploy_model_name,\n",
+        "    machine_type=PREDICTION_MACHINE_TYPE,\n",
+        "    traffic_split={\"0\": 100},\n",
+        "    accelerator_type=PREDICTION_ACCELERATOR_TYPE,\n",
+        "    accelerator_count=1,\n",
+        "    min_replica_count=1,\n",
+        "    max_replica_count=1,\n",
+        ")\n",
+        "\n",
+        "endpoint_id = endpoint.name\n",
+        "print(\"endpoint id is: \", endpoint_id)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vbIW9me1F2RY"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Run predictions\n",
+        "\n",
+        "# endpoint_id was generated in the section above (`Upload and deploy models`).\n",
+        "endpoint_id = endpoint.name\n",
+        "\n",
+        "# The test image file path.\n",
+        "test_filepath = \"\"  # @param {type:\"string\"}\n",
+        "score_threshold = 0.2  # @param {type:\"number\"}\n",
+        "# If the input image is too large, we will resize it for prediction.\n",
+        "instances = get_prediction_instances(test_filepath, new_width=1000)\n",
+        "\n",
+        "# The label map file was generated from the section above (`Convert input data for training`).\n",
+        "label_map = get_label_map(label_map_path)[\"label_map\"]\n",
+        "\n",
+        "predictions, _ = predict_custom_trained_model(\n",
+        "    project=PROJECT_ID, location=REGION, endpoint_id=endpoint_id, instances=instances\n",
+        ")\n",
+        "\n",
+        "img = load_img(test_filepath)\n",
+        "detection_boxes = predictions[0][\"detection_boxes\"]\n",
+        "detection_scores = predictions[0][\"detection_scores\"]\n",
+        "detection_classes_as_text = []\n",
+        "\n",
+        "for detection_class in predictions[0][\"detection_classes\"]:\n",
+        "    detection_classes_as_text.append(label_map[int(detection_class)])\n",
+        "\n",
+        "img = draw_boxes(\n",
+        "    img,\n",
+        "    detection_boxes,\n",
+        "    detection_classes_as_text,\n",
+        "    detection_scores,\n",
+        "    min_score=score_threshold,\n",
+        ")\n",
+        "display_image(img)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kkH2nrpdp4sp"
+      },
+      "source": [
+        "## Clean up"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ax6vQVZhp9pR"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete models.\n",
+        "model.delete()\n",
+        "# Undeploy model and delete endpoint.\n",
+        "endpoint.delete(force=True)\n",
+        "# Delete custom and hpt jobs.\n",
+        "data_converter_custom_job.delete()\n",
+        "train_hpt_job.delete()\n",
+        "model_export_custom_job.delete()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "model_garden_tfvision_image_object_detection.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/notebook_template.ipynb

@@ -162,7 +162,13 @@
       "outputs": [],
       "source": [
         "# Install the packages\n",
-        "! pip3 install --user --upgrade google-cloud-aiplatform"
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    USER = \"--user\"\n",
+        "else:\n",
+        "    USER = \"\"\n",
+        "! pip3 install {USER} --upgrade google-cloud-aiplatform"
       ]
     },
     {

notebooks/official/automl/automl_image_classification_batch_prediction.ipynb

@@ -0,0 +1,884 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "copyright"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "title"
+      },
+      "source": [
+        "# AutoML training image classification model for batch prediction\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_image_classification_batch_prediction.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_image_classification_batch_prediction.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/automl/automl_image_classification_online_prediction.ipynb\">\n",
+        "       <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>\n",
+        "<br/><br/><br/>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "overview:automl"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "\n",
+        "This tutorial demonstrates how to use the Vertex AI SDK to create image classification models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
+        "\n",
+        "Learn more about [Get predictions from an image classification model](https://cloud.google.com/vertex-ai/docs/image-data/classification/get-predictions)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "objective:automl,training,batch_prediction"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you create an AutoML image classification model from a Python script, and then do a batch prediction using the Vertex SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Create a Vertex `Dataset` resource.\n",
+        "- Train the model.\n",
+        "- View the model evaluation.\n",
+        "- Make a batch prediction.\n",
+        "\n",
+        "There is one key difference between using batch prediction and using online prediction:\n",
+        "\n",
+        "* Prediction Service: Does an on-demand prediction for the entire set of instances (i.e., one or more data items) and returns the results in real-time.\n",
+        "\n",
+        "* Batch Prediction Service: Does a queued (batch) prediction for the entire set of instances in the background and stores the results in a Cloud Storage bucket when ready."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:flowers,icn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [Flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of flower an image is from a class of five flowers: daisy, dandelion, rose, sunflower, or tulip."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "costs"
+      },
+      "source": [
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "install_aip:mbsdk"
+      },
+      "source": [
+        "## Installation\n",
+        "\n",
+        "Install the latest version of Vertex AI SDK for Python."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_aip:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "USER_FLAG = \"\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG\n",
+        "\n",
+        "if os.environ[\"IS_TESTING\"]:\n",
+        "    ! pip3 install --upgrade tensorflow $USER_FLAG"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "restart"
+      },
+      "source": [
+        "### Colab only: Uncomment the following cell to restart the kernel"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "D-ZBOjErv5mM"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
+        "\n",
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "before_you_begin:nogpu"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "before_you_begin:nogpu"
+      },
+      "source": [
+        "### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_project_id"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "region"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FvQeFm3Gv5mR"
+      },
+      "source": [
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ad1138a125ea"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ce6043da7b33"
+      },
+      "outputs": [],
+      "source": [
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0367eac06a10"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "21ad4dbb4a61"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c13224697bfb"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bucket:mbsdk"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "Create a storage bucket to store intermediate artifacts such as datasets."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bucket"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_URI = f\"gs://your-bucket-name-unique-{PROJECT_ID}\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_bucket"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "create_bucket"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "setup_vars"
+      },
+      "source": [
+        "### Set up variables\n",
+        "\n",
+        "Next, set up some variables used throughout the tutorial.\n",
+        "### Import libraries and define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_aip:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "import google.cloud.aiplatform as aiplatform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk"
+      },
+      "source": [
+        "## Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "init_aip:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tutorial_start:automl"
+      },
+      "source": [
+        "# Tutorial\n",
+        "\n",
+        "Now you are ready to start creating your own AutoML image classification model."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "import_file:u_dataset,csv"
+      },
+      "source": [
+        "#### Location of Cloud Storage training data.\n",
+        "\n",
+        "Now set the variable `IMPORT_FILE` to the location of the CSV index file in Cloud Storage."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_file:flowers,csv,icn"
+      },
+      "outputs": [],
+      "source": [
+        "IMPORT_FILE = (\n",
+        "    \"gs://cloud-samples-data/vision/automl_classification/flowers/all_data_v2.csv\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "quick_peek:csv"
+      },
+      "source": [
+        "#### Quick peek at your data\n",
+        "\n",
+        "This tutorial uses a version of the Flowers dataset that is stored in a public Cloud Storage bucket, using a CSV index file.\n",
+        "\n",
+        "Start by doing a quick peek at the data. You count the number of examples by counting the number of rows in the CSV index file  (`wc -l`) and then peek at the first few rows."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "quick_peek:csv"
+      },
+      "outputs": [],
+      "source": [
+        "if \"IMPORT_FILES\" in globals():\n",
+        "    FILE = IMPORT_FILES[0]\n",
+        "else:\n",
+        "    FILE = IMPORT_FILE\n",
+        "\n",
+        "count = ! gsutil cat $FILE | wc -l\n",
+        "print(\"Number of Examples\", int(count[0]))\n",
+        "\n",
+        "print(\"First 10 rows\")\n",
+        "! gsutil cat $FILE | head"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_dataset:image,icn"
+      },
+      "source": [
+        "### Create the Dataset\n",
+        "\n",
+        "Next, create the `Dataset` resource using the `create` method for the `ImageDataset` class, which takes the following parameters:\n",
+        "\n",
+        "- `display_name`: The human readable name for the `Dataset` resource.\n",
+        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
+        "- `import_schema_uri`: The data labeling schema for the data items.\n",
+        "\n",
+        "This operation may take several minutes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "create_dataset:image,icn"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = aiplatform.ImageDataset.create(\n",
+        "    display_name=\"Flowers\",\n",
+        "    gcs_source=[IMPORT_FILE],\n",
+        "    import_schema_uri=aiplatform.schema.dataset.ioformat.image.single_label_classification,\n",
+        ")\n",
+        "\n",
+        "print(dataset.resource_name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_automl_pipeline:image,icn"
+      },
+      "source": [
+        "### Create and run training pipeline\n",
+        "\n",
+        "To train an AutoML model, you perform two steps: 1) create a training pipeline, and 2) run the pipeline.\n",
+        "\n",
+        "#### Create training pipeline\n",
+        "\n",
+        "An AutoML training pipeline is created with the `AutoMLImageTrainingJob` class, with the following parameters:\n",
+        "\n",
+        "- `display_name`: The human readable name for the `TrainingJob` resource.\n",
+        "- `prediction_type`: The type task to train the model for.\n",
+        "  - `classification`: An image classification model.\n",
+        "  - `object_detection`: An image object detection model.\n",
+        "- `multi_label`: If a classification task, whether single (`False`) or multi-labeled (`True`).\n",
+        "- `model_type`: The type of model for deployment.\n",
+        "  - `CLOUD`: Deployment on Google Cloud\n",
+        "  - `CLOUD_HIGH_ACCURACY_1`: Optimized for accuracy over latency for deployment on Google Cloud.\n",
+        "  - `CLOUD_LOW_LATENCY_`: Optimized for latency over accuracy for deployment on Google Cloud.\n",
+        "  - `MOBILE_TF_VERSATILE_1`: Deployment on an edge device.\n",
+        "  - `MOBILE_TF_HIGH_ACCURACY_1`:Optimized for accuracy over latency for deployment on an edge device.\n",
+        "  - `MOBILE_TF_LOW_LATENCY_1`: Optimized for latency over accuracy for deployment on an edge device.\n",
+        "- `base_model`: (optional) Transfer learning from existing `Model` resource -- supported for image classification only.\n",
+        "\n",
+        "The instantiated object is the DAG (directed acyclic graph) for the training job."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "create_automl_pipeline:image,icn"
+      },
+      "outputs": [],
+      "source": [
+        "dag = aiplatform.AutoMLImageTrainingJob(\n",
+        "    display_name=\"flowers\",\n",
+        "    prediction_type=\"classification\",\n",
+        "    multi_label=False,\n",
+        "    model_type=\"CLOUD\",\n",
+        "    base_model=None,\n",
+        ")\n",
+        "\n",
+        "print(dag)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "run_automl_pipeline:image"
+      },
+      "source": [
+        "#### Run the training pipeline\n",
+        "\n",
+        "Next, you run the DAG to start the training job by invoking the method `run`, with the following parameters:\n",
+        "\n",
+        "- `dataset`: The `Dataset` resource to train the model.\n",
+        "- `model_display_name`: The human readable name for the trained model.\n",
+        "- `training_fraction_split`: The percentage of the dataset to use for training.\n",
+        "- `test_fraction_split`: The percentage of the dataset to use for test (holdout data).\n",
+        "- `validation_fraction_split`: The percentage of the dataset to use for validation.\n",
+        "- `budget_milli_node_hours`: (optional) Maximum training time specified in unit of millihours (1000 = hour).\n",
+        "- `disable_early_stopping`: If `True`, training maybe completed before using the entire budget if the service believes it cannot further improve on the model objective measurements.\n",
+        "\n",
+        "The `run` method when completed returns the `Model` resource.\n",
+        "\n",
+        "The execution of the training pipeline will take upto 20 minutes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "run_automl_pipeline:image"
+      },
+      "outputs": [],
+      "source": [
+        "model = dag.run(\n",
+        "    dataset=dataset,\n",
+        "    model_display_name=\"flowers\",\n",
+        "    training_fraction_split=0.8,\n",
+        "    validation_fraction_split=0.1,\n",
+        "    test_fraction_split=0.1,\n",
+        "    budget_milli_node_hours=8000,\n",
+        "    disable_early_stopping=False,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "evaluate_the_model:mbsdk"
+      },
+      "source": [
+        "## Review model evaluation scores\n",
+        "\n",
+        "After your model training has finished, you can review the evaluation scores for it using the `list_model_evaluations()` method. This method will return an iterator for each evaluation slice."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "make_prediction"
+      },
+      "source": [
+        "## Send a batch prediction request\n",
+        "\n",
+        "Send a batch prediction to your deployed model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "evaluate_the_model:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "model_evaluations = model.list_model_evaluations()\n",
+        "\n",
+        "for model_evaluation in model_evaluations:\n",
+        "    print(model_evaluation.to_dict())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "get_test_items:batch_prediction"
+      },
+      "source": [
+        "### Get test item(s)\n",
+        "\n",
+        "Now do a batch prediction to your Vertex model. You will use arbitrary examples out of the dataset as a test items. Don't be concerned that the examples were likely used in training the model -- we just want to demonstrate how to make a prediction."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "get_test_items:automl,icn,csv"
+      },
+      "outputs": [],
+      "source": [
+        "test_items = !gsutil cat $IMPORT_FILE | head -n2\n",
+        "if len(str(test_items[0]).split(\",\")) == 3:\n",
+        "    _, test_item_1, test_label_1 = str(test_items[0]).split(\",\")\n",
+        "    _, test_item_2, test_label_2 = str(test_items[1]).split(\",\")\n",
+        "else:\n",
+        "    test_item_1, test_label_1 = str(test_items[0]).split(\",\")\n",
+        "    test_item_2, test_label_2 = str(test_items[1]).split(\",\")\n",
+        "\n",
+        "print(test_item_1, test_label_1)\n",
+        "print(test_item_2, test_label_2)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "copy_test_items:batch_prediction"
+      },
+      "source": [
+        "### Copy test item(s)\n",
+        "\n",
+        "For the batch prediction, copy the test items over to your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "copy_test_items:batch_prediction"
+      },
+      "outputs": [],
+      "source": [
+        "file_1 = test_item_1.split(\"/\")[-1]\n",
+        "file_2 = test_item_2.split(\"/\")[-1]\n",
+        "\n",
+        "! gsutil cp $test_item_1 $BUCKET_URI/$file_1\n",
+        "! gsutil cp $test_item_2 $BUCKET_URI/$file_2\n",
+        "\n",
+        "test_item_1 = BUCKET_URI + \"/\" + file_1\n",
+        "test_item_2 = BUCKET_URI + \"/\" + file_2"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "make_batch_file:automl,image"
+      },
+      "source": [
+        "### Make the batch input file\n",
+        "\n",
+        "Now make a batch input file, which you will store in your local Cloud Storage bucket. The batch input file can be either CSV or JSONL. You will use JSONL in this tutorial. For JSONL file, you make one dictionary entry per line for each data item (instance). The dictionary contains the key/value pairs:\n",
+        "\n",
+        "- `content`: The Cloud Storage path to the image.\n",
+        "- `mime_type`: The content type. In our example, it is a `jpeg` file.\n",
+        "\n",
+        "For example:\n",
+        "\n",
+        "                        {'content': '[your-bucket]/file1.jpg', 'mime_type': 'jpeg'}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "make_batch_file:automl,image"
+      },
+      "outputs": [],
+      "source": [
+        "import json\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "\n",
+        "gcs_input_uri = BUCKET_URI + \"/test.jsonl\"\n",
+        "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
+        "    data = {\"content\": test_item_1, \"mime_type\": \"image/jpeg\"}\n",
+        "    f.write(json.dumps(data) + \"\\n\")\n",
+        "    data = {\"content\": test_item_2, \"mime_type\": \"image/jpeg\"}\n",
+        "    f.write(json.dumps(data) + \"\\n\")\n",
+        "\n",
+        "print(gcs_input_uri)\n",
+        "! gsutil cat $gcs_input_uri"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "batch_request:mbsdk"
+      },
+      "source": [
+        "### Make the batch prediction request\n",
+        "\n",
+        "Now that your Model resource is trained, you can make a batch prediction by invoking the batch_predict() method, with the following parameters:\n",
+        "\n",
+        "- `job_display_name`: The human readable name for the batch prediction job.\n",
+        "- `gcs_source`: A list of one or more batch request input files.\n",
+        "- `gcs_destination_prefix`: The Cloud Storage location for storing the batch prediction resuls.\n",
+        "- `sync`: If set to True, the call will block while waiting for the asynchronous batch job to complete."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "batch_request:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "batch_predict_job = model.batch_predict(\n",
+        "    job_display_name=\"flowers\",\n",
+        "    gcs_source=gcs_input_uri,\n",
+        "    gcs_destination_prefix=BUCKET_URI,\n",
+        "    sync=False,\n",
+        ")\n",
+        "\n",
+        "print(batch_predict_job)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "batch_request_wait:mbsdk"
+      },
+      "source": [
+        "### Wait for completion of batch prediction job\n",
+        "\n",
+        "Next, wait for the batch job to complete. Alternatively, one can set the parameter `sync` to `True` in the `batch_predict()` method to block until the batch prediction job is completed."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "batch_request_wait:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "batch_predict_job.wait()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "get_batch_prediction:mbsdk,icn"
+      },
+      "source": [
+        "### Get the predictions\n",
+        "\n",
+        "Next, get the results from the completed batch prediction job.\n",
+        "\n",
+        "The results are written to the Cloud Storage output bucket you specified in the batch prediction request. You call the method iter_outputs() to get a list of each Cloud Storage file generated with the results. Each file contains one or more prediction requests in a JSON format:\n",
+        "\n",
+        "- `content`: The prediction request.\n",
+        "- `prediction`: The prediction response.\n",
+        " - `ids`: The internal assigned unique identifiers for each prediction request.\n",
+        " - `displayNames`: The class names for each class label.\n",
+        " - `confidences`: The predicted confidence, between 0 and 1, per class label."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "get_batch_prediction:mbsdk,icn"
+      },
+      "outputs": [],
+      "source": [
+        "import json\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "\n",
+        "bp_iter_outputs = batch_predict_job.iter_outputs()\n",
+        "\n",
+        "prediction_results = list()\n",
+        "for blob in bp_iter_outputs:\n",
+        "    if blob.name.split(\"/\")[-1].startswith(\"prediction\"):\n",
+        "        prediction_results.append(blob.name)\n",
+        "\n",
+        "tags = list()\n",
+        "for prediction_result in prediction_results:\n",
+        "    gfile_name = f\"gs://{bp_iter_outputs.bucket.name}/{prediction_result}\"\n",
+        "    with tf.io.gfile.GFile(name=gfile_name, mode=\"r\") as gfile:\n",
+        "        for line in gfile.readlines():\n",
+        "            line = json.loads(line)\n",
+        "            print(line)\n",
+        "            break"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cleanup:mbsdk"
+      },
+      "source": [
+        "# Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cleanup:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "delete_bucket = False\n",
+        "\n",
+        "# Delete the dataset using the Vertex dataset object\n",
+        "dataset.delete()\n",
+        "\n",
+        "# Delete the model using the Vertex model object\n",
+        "model.delete()\n",
+        "\n",
+        "# Delete the AutoML trainig job\n",
+        "dag.delete()\n",
+        "\n",
+        "# Delete the batch prediction job\n",
+        "batch_predict_job.delete()\n",
+        "\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "automl_image_classification_batch_prediction.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/official/automl/automl_image_classification_online_online_prediction.ipynb

@@ -0,0 +1,801 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "copyright"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2020 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "title"
+      },
+      "source": [
+        "# AutoML training image classification model for online prediction\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_image_classification_online_prediction.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_image_classification_online_prediction.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/automl//automl_image_classification_online_prediction.ipynb\">\n",
+        "       <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>\n",
+        "<br/><br/><br/>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "overview:automl"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "\n",
+        "This tutorial demonstrates how to use the Vertex AI SDK to create image classification models and do online prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
+        "\n",
+        "Learn more about [Get predictions from an image classification model](https://cloud.google.com/vertex-ai/docs/image-data/classification/get-predictions)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "objective:automl,training,online_prediction"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you create an AutoML image classification model and deploy for online prediction from a Python script using the Vertex SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Create a Vertex `Dataset` resource.\n",
+        "- Train the model.\n",
+        "- View the model evaluation.\n",
+        "- Deploy the `Model` resource to a serving `Endpoint` resource.\n",
+        "- Make a prediction.\n",
+        "- Undeploy the `Model`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:flowers,icn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [Flowers dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the type of flower an image is from a class of five flowers: daisy, dandelion, rose, sunflower, or tulip."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "costs"
+      },
+      "source": [
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "install_aip:mbsdk"
+      },
+      "source": [
+        "## Installation\n",
+        "\n",
+        "Install the latest version of Vertex AI SDK for Python."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_aip:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "USER_FLAG = \"\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform \\\n",
+        "                         tensorflow $USER_FLAG"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "restart"
+      },
+      "source": [
+        "### Colab only: Uncomment the following cell to restart the kernel"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "D-ZBOjErv5mM"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
+        "\n",
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "before_you_begin:nogpu"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "before_you_begin:nogpu"
+      },
+      "source": [
+        "### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_project_id"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "region"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FvQeFm3Gv5mR"
+      },
+      "source": [
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ad1138a125ea"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ce6043da7b33"
+      },
+      "outputs": [],
+      "source": [
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0367eac06a10"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "21ad4dbb4a61"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c13224697bfb"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bucket:mbsdk"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "Create a storage bucket to store intermediate artifacts such as datasets."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bucket"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_URI = f\"gs://your-bucket-name-unique-{PROJECT_ID}\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_bucket"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "09kHSsKmv5mT"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "setup_vars"
+      },
+      "source": [
+        "### Set up variables\n",
+        "\n",
+        "Next, set up some variables used throughout the tutorial.\n",
+        "### Import libraries and define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_aip:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "import google.cloud.aiplatform as aiplatform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk"
+      },
+      "source": [
+        "## Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "init_aip:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tutorial_start:automl"
+      },
+      "source": [
+        "# Tutorial\n",
+        "\n",
+        "Now you are ready to start creating your own AutoML image classification model."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "import_file:u_dataset,csv"
+      },
+      "source": [
+        "#### Location of Cloud Storage training data.\n",
+        "\n",
+        "Now set the variable `IMPORT_FILE` to the location of the CSV index file in Cloud Storage."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_file:flowers,csv,icn"
+      },
+      "outputs": [],
+      "source": [
+        "IMPORT_FILE = (\n",
+        "    \"gs://cloud-samples-data/vision/automl_classification/flowers/all_data_v2.csv\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "quick_peek:csv"
+      },
+      "source": [
+        "#### Quick peek at your data\n",
+        "\n",
+        "This tutorial uses a version of the Flowers dataset that is stored in a public Cloud Storage bucket, using a CSV index file.\n",
+        "\n",
+        "Start by doing a quick peek at the data. You count the number of examples by counting the number of rows in the CSV index file  (`wc -l`) and then peek at the first few rows."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "quick_peek:csv"
+      },
+      "outputs": [],
+      "source": [
+        "if \"IMPORT_FILES\" in globals():\n",
+        "    FILE = IMPORT_FILES[0]\n",
+        "else:\n",
+        "    FILE = IMPORT_FILE\n",
+        "\n",
+        "count = ! gsutil cat $FILE | wc -l\n",
+        "print(\"Number of Examples\", int(count[0]))\n",
+        "\n",
+        "print(\"First 10 rows\")\n",
+        "! gsutil cat $FILE | head"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_dataset:image,icn"
+      },
+      "source": [
+        "### Create the Dataset\n",
+        "\n",
+        "Next, create the `Dataset` resource using the `create` method for the `ImageDataset` class, which takes the following parameters:\n",
+        "\n",
+        "- `display_name`: The human readable name for the `Dataset` resource.\n",
+        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
+        "- `import_schema_uri`: The data labeling schema for the data items.\n",
+        "\n",
+        "This operation may take several minutes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "create_dataset:image,icn"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = aiplatform.ImageDataset.create(\n",
+        "    display_name=\"Flowers\",\n",
+        "    gcs_source=[IMPORT_FILE],\n",
+        "    import_schema_uri=aiplatform.schema.dataset.ioformat.image.single_label_classification,\n",
+        ")\n",
+        "\n",
+        "print(dataset.resource_name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_automl_pipeline:image,icn"
+      },
+      "source": [
+        "### Create and run training pipeline\n",
+        "\n",
+        "To train an AutoML model, you perform two steps: 1) create a training pipeline, and 2) run the pipeline.\n",
+        "\n",
+        "#### Create training pipeline\n",
+        "\n",
+        "An AutoML training pipeline is created with the `AutoMLImageTrainingJob` class, with the following parameters:\n",
+        "\n",
+        "- `display_name`: The human readable name for the `TrainingJob` resource.\n",
+        "- `prediction_type`: The type task to train the model for.\n",
+        "  - `classification`: An image classification model.\n",
+        "  - `object_detection`: An image object detection model.\n",
+        "- `multi_label`: If a classification task, whether single (`False`) or multi-labeled (`True`).\n",
+        "- `model_type`: The type of model for deployment.\n",
+        "  - `CLOUD`: Deployment on Google Cloud\n",
+        "  - `CLOUD_HIGH_ACCURACY_1`: Optimized for accuracy over latency for deployment on Google Cloud.\n",
+        "  - `CLOUD_LOW_LATENCY_`: Optimized for latency over accuracy for deployment on Google Cloud.\n",
+        "  - `MOBILE_TF_VERSATILE_1`: Deployment on an edge device.\n",
+        "  - `MOBILE_TF_HIGH_ACCURACY_1`:Optimized for accuracy over latency for deployment on an edge device.\n",
+        "  - `MOBILE_TF_LOW_LATENCY_1`: Optimized for latency over accuracy for deployment on an edge device.\n",
+        "- `base_model`: (optional) Transfer learning from existing `Model` resource -- supported for image classification only.\n",
+        "\n",
+        "The instantiated object is the DAG (directed acyclic graph) for the training job."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "create_automl_pipeline:image,icn"
+      },
+      "outputs": [],
+      "source": [
+        "dag = aiplatform.AutoMLImageTrainingJob(\n",
+        "    display_name=\"flowers\",\n",
+        "    prediction_type=\"classification\",\n",
+        "    multi_label=False,\n",
+        "    model_type=\"CLOUD\",\n",
+        "    base_model=None,\n",
+        ")\n",
+        "\n",
+        "print(dag)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "run_automl_pipeline:image"
+      },
+      "source": [
+        "#### Run the training pipeline\n",
+        "\n",
+        "Next, you run the DAG to start the training job by invoking the method `run`, with the following parameters:\n",
+        "\n",
+        "- `dataset`: The `Dataset` resource to train the model.\n",
+        "- `model_display_name`: The human readable name for the trained model.\n",
+        "- `training_fraction_split`: The percentage of the dataset to use for training.\n",
+        "- `test_fraction_split`: The percentage of the dataset to use for test (holdout data).\n",
+        "- `validation_fraction_split`: The percentage of the dataset to use for validation.\n",
+        "- `budget_milli_node_hours`: (optional) Maximum training time specified in unit of millihours (1000 = hour).\n",
+        "- `disable_early_stopping`: If `True`, training maybe completed before using the entire budget if the service believes it cannot further improve on the model objective measurements.\n",
+        "\n",
+        "The `run` method when completed returns the `Model` resource.\n",
+        "\n",
+        "The execution of the training pipeline will take upto 20 minutes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "run_automl_pipeline:image"
+      },
+      "outputs": [],
+      "source": [
+        "model = dag.run(\n",
+        "    dataset=dataset,\n",
+        "    model_display_name=\"flowers\",\n",
+        "    training_fraction_split=0.8,\n",
+        "    validation_fraction_split=0.1,\n",
+        "    test_fraction_split=0.1,\n",
+        "    budget_milli_node_hours=8000,\n",
+        "    disable_early_stopping=False,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "evaluate_the_model:mbsdk"
+      },
+      "source": [
+        "## Review model evaluation scores\n",
+        "\n",
+        "After your model training has finished, you can review the evaluation scores for it using the `list_model_evaluations()` method. This method will return an iterator for each evaluation slice."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "evaluate_the_model:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "model_evaluations = model.list_model_evaluations()\n",
+        "\n",
+        "for model_evaluation in model_evaluations:\n",
+        "    print(model_evaluation.to_dict())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "deploy_model:mbsdk,automatic"
+      },
+      "source": [
+        "## Deploy the model\n",
+        "\n",
+        "Next, deploy your model for online prediction. To deploy the model, you invoke the `deploy` method."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "deploy_model:mbsdk,automatic"
+      },
+      "outputs": [],
+      "source": [
+        "endpoint = model.deploy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "make_prediction"
+      },
+      "source": [
+        "## Send an online prediction request\n",
+        "\n",
+        "Send an online prediction to your deployed model."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "get_test_item"
+      },
+      "source": [
+        "### Get test item\n",
+        "\n",
+        "You will use an arbitrary example out of the dataset as a test item. Don't be concerned that the example was likely used in training the model -- we just want to demonstrate how to make a prediction."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "get_test_item:automl,icn,csv"
+      },
+      "outputs": [],
+      "source": [
+        "test_item = !gsutil cat $IMPORT_FILE | head -n1\n",
+        "if len(str(test_item[0]).split(\",\")) == 3:\n",
+        "    _, test_item, test_label = str(test_item[0]).split(\",\")\n",
+        "else:\n",
+        "    test_item, test_label = str(test_item[0]).split(\",\")\n",
+        "\n",
+        "print(test_item, test_label)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "predict_request:mbsdk,icn"
+      },
+      "source": [
+        "### Make the prediction\n",
+        "\n",
+        "Now that your `Model` resource is deployed to an `Endpoint` resource, you can do online predictions by sending prediction requests to the Endpoint resource.\n",
+        "\n",
+        "#### Request\n",
+        "\n",
+        "Since in this example your test item is in a Cloud Storage bucket, you open and read the contents of the image using `tf.io.gfile.Gfile()`. To pass the test data to the prediction service, you encode the bytes into base64 -- which makes the content safe from modification while transmitting binary data over the network.\n",
+        "\n",
+        "The format of each instance is:\n",
+        "\n",
+        "    { 'content': { 'b64': base64_encoded_bytes } }\n",
+        "\n",
+        "Since the `predict()` method can take multiple items (instances), send your single test item as a list of one test item.\n",
+        "\n",
+        "#### Response\n",
+        "\n",
+        "The response from the `predict()` call is a Python dictionary with the following entries:\n",
+        "\n",
+        "- `ids`: The internal assigned unique identifiers for each prediction request.\n",
+        "- `displayNames`: The class names for each class label.\n",
+        "- `confidences`: The predicted confidence, between 0 and 1, per class label.\n",
+        "- `deployed_model_id`: The Vertex AI identifier for the deployed Model resource which did the predictions."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "predict_request:mbsdk,icn"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "\n",
+        "with tf.io.gfile.GFile(test_item, \"rb\") as f:\n",
+        "    content = f.read()\n",
+        "\n",
+        "# The format of each instance should conform to the deployed model's prediction input schema.\n",
+        "instances = [{\"content\": base64.b64encode(content).decode(\"utf-8\")}]\n",
+        "\n",
+        "prediction = endpoint.predict(instances=instances)\n",
+        "\n",
+        "print(prediction)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "undeploy_model:mbsdk"
+      },
+      "source": [
+        "## Undeploy the model\n",
+        "\n",
+        "When you are done doing predictions, you undeploy the model from the `Endpoint` resouce. This deprovisions all compute resources and ends billing for the deployed model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "undeploy_model:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "endpoint.undeploy_all()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cleanup:mbsdk"
+      },
+      "source": [
+        "# Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cleanup:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "delete_bucket = False\n",
+        "\n",
+        "# Delete the dataset using the Vertex dataset object\n",
+        "dataset.delete()\n",
+        "\n",
+        "try:\n",
+        "    endpoint.undeploy_all()\n",
+        "    endpoint.delete()\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
+        "\n",
+        "# Delete the model using the Vertex model object\n",
+        "model.delete()\n",
+        "\n",
+        "# Delete the AutoML trainig job\n",
+        "dag.delete()\n",
+        "\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "automl_image_classification_online_online_prediction.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/official/automl/automl_image_object_detection_export_edge.ipynb

@@ -0,0 +1,835 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "copyright"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "title"
+      },
+      "source": [
+        "# AutoML training image object detection model for export to edge\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_image_object_detection_export_edge.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_image_object_detection_export_edge.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/automl//automl_image_object_detection_export_edge.ipynb\">\n",
+        "       <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>\n",
+        "<br/><br/><br/>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "overview:automl,export_edge"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "\n",
+        "This tutorial demonstrates how to use the Vertex AI SDK to create image object detection models to export as an Edge model using an AutoML model."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "objective:automl,training,export_edge"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you create an AutoML image object detection model from a Python script using the Vertex SDK, and then export the model as an Edge model in TFLite format. You can alternatively create models with AutoML using the `gcloud` command-line tool or online using the Cloud Console.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- Vertex AI `Datasets`\n",
+        "- AutoML Image\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Create a Vertex `Dataset` resource.\n",
+        "- Train the model.\n",
+        "- Export the `Edge` model from the `Model` resource to Cloud Storage.\n",
+        "- Download the model locally.\n",
+        "- Make a local prediction."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:salads,iod"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the Salads category of the [OpenImages dataset](https://www.tensorflow.org/datasets/catalog/open_images_v4) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the bounding box locations and corresponding type of salad items in an image from a class of five items: salad, seafood, tomato, baked goods, or cheese."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "costs"
+      },
+      "source": [
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "install_aip:mbsdk"
+      },
+      "source": [
+        "## Installation\n",
+        "\n",
+        "Install the latest version of Vertex AI SDK for Python."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_aip:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# Google Cloud Notebook\n",
+        "if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "    USER_FLAG = \"--user\"\n",
+        "else:\n",
+        "    USER_FLAG = \"\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG\n",
+        "\n",
+        "if os.environ[\"IS_TESTING\"]:\n",
+        "    ! pip3 install --upgrade tensorflow $USER_FLAG"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "restart"
+      },
+      "source": [
+        "### Colab only: Uncomment the following cell to restart the kernel"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "D-ZBOjErv5mM"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
+        "\n",
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "before_you_begin:nogpu"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "before_you_begin:nogpu"
+      },
+      "source": [
+        "### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_project_id"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "region"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FvQeFm3Gv5mR"
+      },
+      "source": [
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ad1138a125ea"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ce6043da7b33"
+      },
+      "outputs": [],
+      "source": [
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0367eac06a10"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "21ad4dbb4a61"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c13224697bfb"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bucket:mbsdk"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "Create a storage bucket to store intermediate artifacts such as datasets."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bucket"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_URI = f\"gs://your-bucket-name-unique-{PROJECT_ID}\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_bucket"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "create_bucket"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "setup_vars"
+      },
+      "source": [
+        "### Set up variables\n",
+        "\n",
+        "Next, set up some variables used throughout the tutorial.\n",
+        "### Import libraries and define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_aip:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "import google.cloud.aiplatform as aiplatform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk"
+      },
+      "source": [
+        "## Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "init_aip:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tutorial_start:automl"
+      },
+      "source": [
+        "# Tutorial\n",
+        "\n",
+        "Now you are ready to start creating your own AutoML image object detection model."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "import_file:u_dataset,csv"
+      },
+      "source": [
+        "#### Location of Cloud Storage training data.\n",
+        "\n",
+        "Now set the variable `IMPORT_FILE` to the location of the CSV index file in Cloud Storage."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_file:salads,csv,iod"
+      },
+      "outputs": [],
+      "source": [
+        "IMPORT_FILE = \"gs://cloud-samples-data/vision/salads.csv\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "quick_peek:csv"
+      },
+      "source": [
+        "#### Quick peek at your data\n",
+        "\n",
+        "This tutorial uses a version of the Salads dataset that is stored in a public Cloud Storage bucket, using a CSV index file.\n",
+        "\n",
+        "Start by doing a quick peek at the data. You count the number of examples by counting the number of rows in the CSV index file  (`wc -l`) and then peek at the first few rows."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "quick_peek:csv"
+      },
+      "outputs": [],
+      "source": [
+        "if \"IMPORT_FILES\" in globals():\n",
+        "    FILE = IMPORT_FILES[0]\n",
+        "else:\n",
+        "    FILE = IMPORT_FILE\n",
+        "\n",
+        "count = ! gsutil cat $FILE | wc -l\n",
+        "print(\"Number of Examples\", int(count[0]))\n",
+        "\n",
+        "print(\"First 10 rows\")\n",
+        "! gsutil cat $FILE | head"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_dataset:image,iod"
+      },
+      "source": [
+        "### Create the Dataset\n",
+        "\n",
+        "Next, create the `Dataset` resource using the `create` method for the `ImageDataset` class, which takes the following parameters:\n",
+        "\n",
+        "- `display_name`: The human readable name for the `Dataset` resource.\n",
+        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
+        "- `import_schema_uri`: The data labeling schema for the data items.\n",
+        "\n",
+        "This operation may take several minutes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "create_dataset:image,iod"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = aiplatform.ImageDataset.create(\n",
+        "    display_name=\"Salads\",\n",
+        "    gcs_source=[IMPORT_FILE],\n",
+        "    import_schema_uri=aiplatform.schema.dataset.ioformat.image.bounding_box,\n",
+        ")\n",
+        "\n",
+        "print(dataset.resource_name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_automl_pipeline:image,edge,iod"
+      },
+      "source": [
+        "### Create and run training pipeline\n",
+        "\n",
+        "To train an AutoML model, you perform two steps: 1) create a training pipeline, and 2) run the pipeline.\n",
+        "\n",
+        "#### Create training pipeline\n",
+        "\n",
+        "An AutoML training pipeline is created with the `AutoMLImageTrainingJob` class, with the following parameters:\n",
+        "\n",
+        "- `display_name`: The human readable name for the `TrainingJob` resource.\n",
+        "- `prediction_type`: The type task to train the model for.\n",
+        "  - `classification`: An image classification model.\n",
+        "  - `object_detection`: An image object detection model.\n",
+        "- `multi_label`: If a classification task, whether single (`False`) or multi-labeled (`True`).\n",
+        "- `model_type`: The type of model for deployment.\n",
+        "  - `CLOUD`: Deployment on Google Cloud\n",
+        "  - `CLOUD_HIGH_ACCURACY_1`: Optimized for accuracy over latency for deployment on Google Cloud.\n",
+        "  - `CLOUD_LOW_LATENCY_`: Optimized for latency over accuracy for deployment on Google Cloud.\n",
+        "  - `MOBILE_TF_VERSATILE_1`: Deployment on an edge device.\n",
+        "  - `MOBILE_TF_HIGH_ACCURACY_1`:Optimized for accuracy over latency for deployment on an edge device.\n",
+        "  - `MOBILE_TF_LOW_LATENCY_1`: Optimized for latency over accuracy for deployment on an edge device.\n",
+        "- `base_model`: (optional) Transfer learning from existing `Model` resource -- supported for image classification only.\n",
+        "\n",
+        "The instantiated object is the DAG (directed acyclic graph) for the training job."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "create_automl_pipeline:image,edge,iod"
+      },
+      "outputs": [],
+      "source": [
+        "dag = aiplatform.AutoMLImageTrainingJob(\n",
+        "    display_name=\"salads\",\n",
+        "    prediction_type=\"object_detection\",\n",
+        "    multi_label=False,\n",
+        "    model_type=\"MOBILE_TF_LOW_LATENCY_1\",\n",
+        "    base_model=None,\n",
+        ")\n",
+        "\n",
+        "print(dag)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "run_automl_pipeline:image"
+      },
+      "source": [
+        "#### Run the training pipeline\n",
+        "\n",
+        "Next, you run the DAG to start the training job by invoking the method `run`, with the following parameters:\n",
+        "\n",
+        "- `dataset`: The `Dataset` resource to train the model.\n",
+        "- `model_display_name`: The human readable name for the trained model.\n",
+        "- `training_fraction_split`: The percentage of the dataset to use for training.\n",
+        "- `test_fraction_split`: The percentage of the dataset to use for test (holdout data).\n",
+        "- `validation_fraction_split`: The percentage of the dataset to use for validation.\n",
+        "- `budget_milli_node_hours`: (optional) Maximum training time specified in unit of millihours (1000 = hour).\n",
+        "- `disable_early_stopping`: If `True`, training maybe completed before using the entire budget if the service believes it cannot further improve on the model objective measurements.\n",
+        "\n",
+        "The `run` method when completed returns the `Model` resource.\n",
+        "\n",
+        "The execution of the training pipeline will take upto 60 minutes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "run_automl_pipeline:image"
+      },
+      "outputs": [],
+      "source": [
+        "model = dag.run(\n",
+        "    dataset=dataset,\n",
+        "    model_display_name=\"salads\",\n",
+        "    training_fraction_split=0.8,\n",
+        "    validation_fraction_split=0.1,\n",
+        "    test_fraction_split=0.1,\n",
+        "    budget_milli_node_hours=20000,\n",
+        "    disable_early_stopping=False,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "evaluate_the_model:mbsdk"
+      },
+      "source": [
+        "## Review model evaluation scores\n",
+        "\n",
+        "After your model training has finished, you can review the evaluation scores for it using the `list_model_evaluations()` method. This method will return an iterator for each evaluation slice."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "evaluate_the_model:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "model_evaluations = model.list_model_evaluations()\n",
+        "\n",
+        "for model_evaluation in model_evaluations:\n",
+        "    print(model_evaluation.to_dict())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "export_model:mbsdk,image"
+      },
+      "source": [
+        "## Export as Edge model\n",
+        "\n",
+        "You can export an AutoML image object detection model as a `Edge` model which you can then custom deploy to an edge device or download locally. Use the method `export_model()` to export the model to Cloud Storage, which takes the following parameters:\n",
+        "\n",
+        "- `artifact_destination`: The Cloud Storage location to store the SavedFormat model artifacts to.\n",
+        "- `export_format_id`: The format to save the model format as. For AutoML image object detection there is just one option:\n",
+        "   - `tf-saved-model`: TensorFlow SavedFormat for deployment to a container.\n",
+        "   - `tflite`: TensorFlow Lite for deployment to an edge or mobile device.\n",
+        "   - `edgetpu-tflite`: TensorFlow Lite for TPU\n",
+        "   - `tf-js`: TensorFlow for web client\n",
+        "   - `coral-ml`: for Coral devices\n",
+        "\n",
+        "- `sync`: Whether to perform operational sychronously or asynchronously."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "export_model:mbsdk,image"
+      },
+      "outputs": [],
+      "source": [
+        "response = model.export_model(\n",
+        "    artifact_destination=BUCKET_URI, export_format_id=\"tflite\", sync=True\n",
+        ")\n",
+        "\n",
+        "model_package = response[\"artifactOutputUri\"]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "download_model_artifacts:tflite"
+      },
+      "source": [
+        "#### Download the TFLite model artifacts\n",
+        "\n",
+        "Now that you have an exported TFLite version of your model, you can test the exported model locally, but first downloading it from Cloud Storage."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "download_model_artifacts:tflite"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls $model_package\n",
+        "# Download the model artifacts\n",
+        "! gsutil cp -r $model_package tflite\n",
+        "\n",
+        "tflite_path = \"tflite/model.tflite\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "instantiate_tflite_interpreter"
+      },
+      "source": [
+        "#### Instantiate a TFLite interpreter\n",
+        "\n",
+        "The TFLite version of the model is not a TensorFlow SavedModel format. You cannot directly use methods like predict(). Instead, one uses the TFLite interpreter. You must first setup the interpreter for the TFLite model as follows:\n",
+        "\n",
+        "- Instantiate an TFLite interpreter for the TFLite model.\n",
+        "- Instruct the interpreter to allocate input and output tensors for the model.\n",
+        "- Get detail information about the models input and output tensors that will need to be known for prediction."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "instantiate_tflite_interpreter"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "\n",
+        "interpreter = tf.lite.Interpreter(model_path=tflite_path)\n",
+        "interpreter.allocate_tensors()\n",
+        "\n",
+        "input_details = interpreter.get_input_details()\n",
+        "output_details = interpreter.get_output_details()\n",
+        "input_shape = input_details[0][\"shape\"]\n",
+        "\n",
+        "print(\"input tensor shape\", input_shape)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "get_test_item"
+      },
+      "source": [
+        "### Get test item\n",
+        "\n",
+        "You will use an arbitrary example out of the dataset as a test item. Don't be concerned that the example was likely used in training the model -- we just want to demonstrate how to make a prediction."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "get_test_item:image,224x224"
+      },
+      "outputs": [],
+      "source": [
+        "test_items = ! gsutil cat $IMPORT_FILE | head -n1\n",
+        "test_item = test_items[0].split(\",\")[0]\n",
+        "\n",
+        "with tf.io.gfile.GFile(test_item, \"rb\") as f:\n",
+        "    content = f.read()\n",
+        "test_image = tf.io.decode_jpeg(content)\n",
+        "print(\"test image shape\", test_image.shape)\n",
+        "\n",
+        "test_image = tf.image.resize(test_image, (192, 192))\n",
+        "print(\"test image shape\", test_image.shape, test_image.dtype)\n",
+        "\n",
+        "test_image = tf.cast(test_image, dtype=tf.uint8).numpy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "invoke_tflite_interpreter"
+      },
+      "source": [
+        "#### Make a prediction with TFLite model\n",
+        "\n",
+        "Finally, you do a prediction using your TFLite model, as follows:\n",
+        "\n",
+        "- Convert the test image into a batch of a single image (`np.expand_dims`)\n",
+        "- Set the input tensor for the interpreter to your batch of a single image (`data`).\n",
+        "- Invoke the interpreter.\n",
+        "- Retrieve the softmax probabilities for the prediction (`get_tensor`).\n",
+        "- Determine which label had the highest probability (`np.argmax`)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "invoke_tflite_interpreter"
+      },
+      "outputs": [],
+      "source": [
+        "import numpy as np\n",
+        "\n",
+        "data = np.expand_dims(test_image, axis=0)\n",
+        "\n",
+        "interpreter.set_tensor(input_details[0][\"index\"], data)\n",
+        "\n",
+        "interpreter.invoke()\n",
+        "\n",
+        "softmax = interpreter.get_tensor(output_details[0][\"index\"])\n",
+        "\n",
+        "label = np.argmax(softmax)\n",
+        "\n",
+        "print(label)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cleanup:mbsdk"
+      },
+      "source": [
+        "# Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cleanup:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "delete_bucket = False\n",
+        "\n",
+        "# Delete the dataset using the Vertex dataset object\n",
+        "dataset.delete()\n",
+        "\n",
+        "# Delete the model using the Vertex model object\n",
+        "model.delete()\n",
+        "\n",
+        "# Delete the AutoML trainig job\n",
+        "dag.delete()\n",
+        "\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "automl_image_object_detection_export_edge.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/official/automl/automl_image_object_detection_online_prediction.ipynb

@@ -0,0 +1,815 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "copyright"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "title"
+      },
+      "source": [
+        "# AutoML training image object detection model for online prediction\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_image_object_detection_online_prediction.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_image_object_detection_online_prediction.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/automl/automl_image_object_detection_online_prediction.ipynb\">\n",
+        "       <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>\n",
+        "<br/><br/><br/>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "overview:automl"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "\n",
+        "This tutorial demonstrates how to use the Vertex AI SDK to create image object detection models and do online prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
+        "\n",
+        "Learn more about [Object detection for image data](https://cloud.google.com/vertex-ai/docs/training-overview#object_detection_for_images)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "objective:automl,training,online_prediction"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you create an AutoML image object detection model and deploy for online prediction from a Python script using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- Vertex AI `Datasets`\n",
+        "- AutoML Image\n",
+        "- Vertex AI `Model Registry`\n",
+        "- Vertex AI `Predictions`\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Create a Vertex `Dataset` resource.\n",
+        "- Train the model.\n",
+        "- View the model evaluation.\n",
+        "- Deploy the `Model` resource to a serving `Endpoint` resource.\n",
+        "- Make a prediction.\n",
+        "- Undeploy the `Model`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:salads,iod"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the Salads category of the [OpenImages dataset](https://www.tensorflow.org/datasets/catalog/open_images_v4) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the bounding box locations and corresponding type of salad items in an image from a class of five items: salad, seafood, tomato, baked goods, or cheese."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "costs"
+      },
+      "source": [
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "install_aip:mbsdk"
+      },
+      "source": [
+        "## Installation\n",
+        "\n",
+        "Install the latest version of Vertex AI SDK for Python."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_aip:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "USER_FLAG = \"\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform \\\n",
+        "                         tensorflow $USER_FLAG\n",
+        "\n",
+        "if os.environ[\"IS_TESTING\"]:\n",
+        "    ! pip3 install --upgrade tensorflow $USER_FLAG"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "restart"
+      },
+      "source": [
+        "### Colab only: Uncomment the following cell to restart the kernel"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "D-ZBOjErv5mM"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
+        "\n",
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "before_you_begin:nogpu"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "before_you_begin:nogpu"
+      },
+      "source": [
+        "### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_project_id"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "region"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gcp_authenticate"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FvQeFm3Gv5mR"
+      },
+      "source": [
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ad1138a125ea"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ce6043da7b33"
+      },
+      "outputs": [],
+      "source": [
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0367eac06a10"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "21ad4dbb4a61"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c13224697bfb"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bucket:mbsdk"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "Create a storage bucket to store intermediate artifacts such as datasets."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bucket"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_URI = f\"gs://your-bucket-name-unique-{PROJECT_ID}\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_bucket"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "create_bucket"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "setup_vars"
+      },
+      "source": [
+        "### Set up variables\n",
+        "\n",
+        "Next, set up some variables used throughout the tutorial.\n",
+        "### Import libraries and define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_aip:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "import google.cloud.aiplatform as aiplatform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk"
+      },
+      "source": [
+        "## Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "init_aip:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tutorial_start:automl"
+      },
+      "source": [
+        "# Tutorial\n",
+        "\n",
+        "Now you are ready to start creating your own AutoML image object detection model."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "import_file:u_dataset,csv"
+      },
+      "source": [
+        "#### Location of Cloud Storage training data.\n",
+        "\n",
+        "Now set the variable `IMPORT_FILE` to the location of the CSV index file in Cloud Storage."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_file:salads,csv,iod"
+      },
+      "outputs": [],
+      "source": [
+        "IMPORT_FILE = \"gs://cloud-samples-data/vision/salads.csv\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "quick_peek:csv"
+      },
+      "source": [
+        "#### Quick peek at your data\n",
+        "\n",
+        "This tutorial uses a version of the Salads dataset that is stored in a public Cloud Storage bucket, using a CSV index file.\n",
+        "\n",
+        "Start by doing a quick peek at the data. You count the number of examples by counting the number of rows in the CSV index file  (`wc -l`) and then peek at the first few rows."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "quick_peek:csv"
+      },
+      "outputs": [],
+      "source": [
+        "if \"IMPORT_FILES\" in globals():\n",
+        "    FILE = IMPORT_FILES[0]\n",
+        "else:\n",
+        "    FILE = IMPORT_FILE\n",
+        "\n",
+        "count = ! gsutil cat $FILE | wc -l\n",
+        "print(\"Number of Examples\", int(count[0]))\n",
+        "\n",
+        "print(\"First 10 rows\")\n",
+        "! gsutil cat $FILE | head"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_dataset:image,iod"
+      },
+      "source": [
+        "### Create the Dataset\n",
+        "\n",
+        "Next, create the `Dataset` resource using the `create` method for the `ImageDataset` class, which takes the following parameters:\n",
+        "\n",
+        "- `display_name`: The human readable name for the `Dataset` resource.\n",
+        "- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
+        "- `import_schema_uri`: The data labeling schema for the data items.\n",
+        "\n",
+        "This operation may take several minutes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "create_dataset:image,iod"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = aiplatform.ImageDataset.create(\n",
+        "    display_name=\"Salads\",\n",
+        "    gcs_source=[IMPORT_FILE],\n",
+        "    import_schema_uri=aiplatform.schema.dataset.ioformat.image.bounding_box,\n",
+        ")\n",
+        "\n",
+        "print(dataset.resource_name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_automl_pipeline:image,iod"
+      },
+      "source": [
+        "### Create and run training pipeline\n",
+        "\n",
+        "To train an AutoML model, you perform two steps: 1) create a training pipeline, and 2) run the pipeline.\n",
+        "\n",
+        "#### Create training pipeline\n",
+        "\n",
+        "An AutoML training pipeline is created with the `AutoMLImageTrainingJob` class, with the following parameters:\n",
+        "\n",
+        "- `display_name`: The human readable name for the `TrainingJob` resource.\n",
+        "- `prediction_type`: The type task to train the model for.\n",
+        "  - `classification`: An image classification model.\n",
+        "  - `object_detection`: An image object detection model.\n",
+        "- `multi_label`: If a classification task, whether single (`False`) or multi-labeled (`True`).\n",
+        "- `model_type`: The type of model for deployment.\n",
+        "  - `CLOUD`: Deployment on Google Cloud\n",
+        "  - `CLOUD_HIGH_ACCURACY_1`: Optimized for accuracy over latency for deployment on Google Cloud.\n",
+        "  - `CLOUD_LOW_LATENCY_`: Optimized for latency over accuracy for deployment on Google Cloud.\n",
+        "  - `MOBILE_TF_VERSATILE_1`: Deployment on an edge device.\n",
+        "  - `MOBILE_TF_HIGH_ACCURACY_1`:Optimized for accuracy over latency for deployment on an edge device.\n",
+        "  - `MOBILE_TF_LOW_LATENCY_1`: Optimized for latency over accuracy for deployment on an edge device.\n",
+        "- `base_model`: (optional) Transfer learning from existing `Model` resource -- supported for image classification only.\n",
+        "\n",
+        "The instantiated object is the DAG (directed acyclic graph) for the training job."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "create_automl_pipeline:image,iod"
+      },
+      "outputs": [],
+      "source": [
+        "dag = aiplatform.AutoMLImageTrainingJob(\n",
+        "    display_name=\"salads\",\n",
+        "    prediction_type=\"object_detection\",\n",
+        "    multi_label=False,\n",
+        "    model_type=\"CLOUD\",\n",
+        "    base_model=None,\n",
+        ")\n",
+        "\n",
+        "print(dag)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "run_automl_pipeline:image"
+      },
+      "source": [
+        "#### Run the training pipeline\n",
+        "\n",
+        "Next, you run the DAG to start the training job by invoking the method `run`, with the following parameters:\n",
+        "\n",
+        "- `dataset`: The `Dataset` resource to train the model.\n",
+        "- `model_display_name`: The human readable name for the trained model.\n",
+        "- `training_fraction_split`: The percentage of the dataset to use for training.\n",
+        "- `test_fraction_split`: The percentage of the dataset to use for test (holdout data).\n",
+        "- `validation_fraction_split`: The percentage of the dataset to use for validation.\n",
+        "- `budget_milli_node_hours`: (optional) Maximum training time specified in unit of millihours (1000 = hour).\n",
+        "- `disable_early_stopping`: If `True`, training maybe completed before using the entire budget if the service believes it cannot further improve on the model objective measurements.\n",
+        "\n",
+        "The `run` method when completed returns the `Model` resource.\n",
+        "\n",
+        "The execution of the training pipeline will take upto 60 minutes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "run_automl_pipeline:image"
+      },
+      "outputs": [],
+      "source": [
+        "model = dag.run(\n",
+        "    dataset=dataset,\n",
+        "    model_display_name=\"salads\",\n",
+        "    training_fraction_split=0.8,\n",
+        "    validation_fraction_split=0.1,\n",
+        "    test_fraction_split=0.1,\n",
+        "    budget_milli_node_hours=20000,\n",
+        "    disable_early_stopping=False,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "evaluate_the_model:mbsdk"
+      },
+      "source": [
+        "## Review model evaluation scores\n",
+        "\n",
+        "After your model training has finished, you can review the evaluation scores for it using the `list_model_evaluations()` method. This method will return an iterator for each evaluation slice."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "evaluate_the_model:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "model_evaluations = model.list_model_evaluations()\n",
+        "\n",
+        "for model_evaluation in model_evaluations:\n",
+        "    print(model_evaluation.to_dict())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "deploy_model:mbsdk,automatic"
+      },
+      "source": [
+        "## Deploy the model\n",
+        "\n",
+        "Next, deploy your model for online prediction. To deploy the model, you invoke the `deploy` method."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "deploy_model:mbsdk,automatic"
+      },
+      "outputs": [],
+      "source": [
+        "endpoint = model.deploy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "make_prediction"
+      },
+      "source": [
+        "## Send an online prediction request\n",
+        "\n",
+        "Send an online prediction to your deployed model."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "get_test_item"
+      },
+      "source": [
+        "### Get test item\n",
+        "\n",
+        "You will use an arbitrary example out of the dataset as a test item. Don't be concerned that the example was likely used in training the model -- we just want to demonstrate how to make a prediction."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "get_test_item:automl,iod,csv"
+      },
+      "outputs": [],
+      "source": [
+        "test_items = !gsutil cat $IMPORT_FILE | head -n1\n",
+        "cols = str(test_items[0]).split(\",\")\n",
+        "if len(cols) == 11:\n",
+        "    test_item = str(cols[1])\n",
+        "    test_label = str(cols[2])\n",
+        "else:\n",
+        "    test_item = str(cols[0])\n",
+        "    test_label = str(cols[1])\n",
+        "\n",
+        "print(test_item, test_label)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "predict_request:mbsdk,iod"
+      },
+      "source": [
+        "### Make the prediction\n",
+        "\n",
+        "Now that your `Model` resource is deployed to an `Endpoint` resource, you can do online predictions by sending prediction requests to the Endpoint resource.\n",
+        "\n",
+        "#### Request\n",
+        "\n",
+        "Since in this example your test item is in a Cloud Storage bucket, you open and read the contents of the image using `tf.io.gfile.Gfile()`. To pass the test data to the prediction service, you encode the bytes into base64 -- which makes the content safe from modification while transmitting binary data over the network.\n",
+        "\n",
+        "The format of each instance is:\n",
+        "\n",
+        "    { 'content': { 'b64': base64_encoded_bytes } }\n",
+        "\n",
+        "Since the `predict()` method can take multiple items (instances), send your single test item as a list of one test item.\n",
+        "\n",
+        "#### Response\n",
+        "\n",
+        "The response from the `predict()` call is a Python dictionary with the following entries:\n",
+        "\n",
+        "- `ids`: The internal assigned unique identifiers for each prediction request.\n",
+        "- `displayNames`: The class names for each class label.\n",
+        "- `confidences`: The predicted confidence, between 0 and 1, per class label.\n",
+        "- `bboxes`: The bounding box of each detected object.\n",
+        "- `deployed_model_id`: The Vertex AI identifier for the deployed Model resource which did the predictions."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "predict_request:mbsdk,iod"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "\n",
+        "with tf.io.gfile.GFile(test_item, \"rb\") as f:\n",
+        "    content = f.read()\n",
+        "\n",
+        "# The format of each instance should conform to the deployed model's prediction input schema.\n",
+        "instances = [{\"content\": base64.b64encode(content).decode(\"utf-8\")}]\n",
+        "\n",
+        "prediction = endpoint.predict(instances=instances)\n",
+        "\n",
+        "print(prediction)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "undeploy_model:mbsdk"
+      },
+      "source": [
+        "## Undeploy the model\n",
+        "\n",
+        "When you are done doing predictions, you undeploy the model from the `Endpoint` resource. This deprovisions all compute resources and ends billing for the deployed model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "undeploy_model:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "endpoint.undeploy_all()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cleanup:mbsdk"
+      },
+      "source": [
+        "# Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cleanup:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "delete_bucket = False\n",
+        "\n",
+        "# Delete the dataset using the Vertex dataset object\n",
+        "dataset.delete()\n",
+        "\n",
+        "try:\n",
+        "    endpoint.undeploy_all()\n",
+        "    endpoint.delete()\n",
+        "except Exception as e:\n",
+        "    print(e)\n",
+        "\n",
+        "# Delete the model using the Vertex model object\n",
+        "model.delete()\n",
+        "\n",
+        "# Delete the AutoML trainig job\n",
+        "dag.delete()\n",
+        "\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "automl_image_object_detection_online_prediction.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/official/automl/sdk_automl_forecasting_hierarchical_batch.ipynb

@@ -87,9 +87,7 @@
         "- Create a Vertex AI `TimeSeriesDataset` resource.\n",
         "- Train the model.\n",
         "- View the model evaluation.\n",
-        "- Deploy the `Model` resource to a serving `Endpoint` resource.\n",
-        "- Make a prediction.\n",
-        "- Undeploy the `Model`."
+        "- Make a batch prediction."
       ]
     },
     {

notebooks/official/experiments/comparing_local_trained_models.ipynb

@@ -194,7 +194,8 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "! pip3 install {USER_FLAG} tensorflow==2.8 google-cloud-aiplatform -q --no-warn-conflicts"
+        "! pip3 install {USER_FLAG} tensorflow==2.8 google-cloud-aiplatform -q --no-warn-conflicts\n",
+        "! pip3 install {USER_FLAG} protobuf==3.20.3"
       ]
     },
     {

notebooks/official/feature_store/feature_store_streaming_ingestion_sdk.ipynb

@@ -152,6 +152,7 @@
         "                         google-cloud-bigquery\\\n",
         "                         numpy\\\n",
         "                         pandas\\\n",
+        "                         db-dtypes\\\n",
         "                         pyarrow -q"
       ]
     },

notebooks/official/feature_store/sdk-feature-store.ipynb

@@ -47,7 +47,7 @@
         "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/feature_store/sdk-feature-store.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
-        "    </a>                                                                                           \n",
+        "    </a>\n",
         "  </td>\n",
         "</table>"
       ]
@@ -107,7 +107,7 @@
         "id": "tvgnzT1CKxrO"
       },
       "source": [
-        "### Costs \n",
+        "### Costs\n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
         "\n",
@@ -125,59 +125,232 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "ze4-nDLfK4pw"
+        "id": "s3Jje0B5zglA"
       },
       "source": [
-        "### Set up your local development environment\n",
+        "## Installation\n",
         "\n",
-        "**If you are using Colab or Vertex AI Workbench notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step."
+        "Install the following packages required to execute this notebook."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MW_NeIHMzjoZ"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install --upgrade google-cloud-aiplatform -q"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "gCuSR8GkAgzl"
+        "id": "GlWoVi7xz1TL"
       },
       "source": [
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
+        "### Colab only: Uncomment the following cell to restart the kernel."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CFS6OPNWz3KZ"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
         "\n",
-        "* The Google Cloud SDK\n",
-        "* Git\n",
-        "* Python 3\n",
-        "* virtualenv\n",
-        "* Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Google Cloud guide to [setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
-        "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
-        "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
-        "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
-        "\n",
-        "1. To install Jupyter, run `pip install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
-        "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "1. Open this notebook in the Jupyter Notebook dashboard."
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "lWEdiXsJg0XY"
+        "id": "7RMhe6650CyB"
       },
       "source": [
-        "## Before you begin"
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "3. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). {TODO: Update the APIs needed for your tutorial. Edit the API names, and update the link to append the API IDs, separating each one with a comma. For example, container.googleapis.com,cloudbuild.googleapis.com}\n",
+        "\n",
+        "4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "T7C_dgnR0L_l"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WbSe_XFH0NjL"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ybtwdOp40TVK"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oLUOopdB0UkU"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "G_ZkpZnv0a0b"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rfsExLao0b49"
+      },
+      "source": [
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ovUeYbbM0nmK"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "l_AmeEXr0pE1"
+      },
+      "outputs": [],
+      "source": [
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fsl-OPfF0sUO"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mOh0DLZP0vUI"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qML_uytf0ymm"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "VVRl2Isi02ZG"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "Create a storage bucket to store intermediate artifacts such as datasets."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CtiQt7ST06f1"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_URI = \"gs://your-bucket-name-unique\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qBr6o7cC1AEj"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "R4QWPo2V1BP0"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
       ]
     },
     {
@@ -195,295 +368,34 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "2b4ef9b72d43"
+        "id": "xuQ4jQTb1Jbc"
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q"
+        "from google.cloud import aiplatform\n",
+        "from google.cloud.aiplatform import Feature, Featurestore"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "hhq5zEbGg0XX"
+        "id": "dOqQGVoO1Kw-"
       },
       "source": [
-        "### Restart the kernel\n",
+        "### Initialize Vertex AI SDK for Python\n",
         "\n",
-        "After you install the SDK, you need to restart the notebook kernel so it can find the packages. You can restart kernel from *Kernel -> Restart Kernel*, or by running the following:"
+        "Initialize the Vertex AI SDK for Python for your project."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "EzrelQZ22IZj"
+        "id": "PWaQMlJ71N4e"
       },
       "outputs": [],
       "source": [
-        "# Automatically restart kernel after installs\n",
-        "import os\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "BF1j6f9HApxa"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API and the Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
-        "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below, and then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "WReHDGG5g0XY"
-      },
-      "source": [
-        "#### Set your project ID\n",
-        "\n",
-        "**If you don't know your project ID**, you can get your project ID using `gcloud`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "oM1iC_MfAts1"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "PROJECT_ID = \"\"\n",
-        "\n",
-        "# Get your Google Cloud project ID from gcloud\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "riG_qUokg0XZ"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
-        "print(\"Project ID: \", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "250cb8c648d5"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "region"
-      },
-      "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook. The following regions are supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "region"
-      },
-      "outputs": [],
-      "source": [
-        "REGION = \"[your-region]\"  # @param {type:\"string\"}\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "timestamp"
-      },
-      "source": [
-        "#### UUID\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name conflicts between users on resources created, you create a UUID for each instance session, and append it onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "timestamp"
-      },
-      "outputs": [],
-      "source": [
-        "import random\n",
-        "import string\n",
-        "\n",
-        "\n",
-        "# Generate a UUID of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dr--iN2kAylZ"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "**If you are using Vertex AI Workbench notebooks**, your environment is already\n",
-        "authenticated."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "sBCra4QMA2wR"
-      },
-      "source": [
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click **Create**. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "PyQmSRbKA8r-"
-      },
-      "outputs": [],
-      "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the following string with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "XoEqT2Y4DJmf"
-      },
-      "source": [
-        "### Import libraries and define constants"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "Cdct_Lm7x2I_"
-      },
-      "outputs": [],
-      "source": [
-        "from google.cloud.aiplatform import Feature, Featurestore\n",
-        "\n",
-        "FEATURESTORE_ID = \"movie_prediction\" + UUID\n",
-        "INPUT_CSV_FILE = \"gs://cloud-samples-data-us-central1/vertex-ai/feature-store/datasets/movie_prediction.csv\"\n",
-        "ONLINE_STORE_FIXED_NODE_COUNT = 1"
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -516,6 +428,19 @@
         "## Create featurestore and define schemas"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Cdct_Lm7x2I_"
+      },
+      "outputs": [],
+      "source": [
+        "FEATURESTORE_ID = \"movie_prediction_unique\"\n",
+        "INPUT_CSV_FILE = \"gs://cloud-samples-data-us-central1/vertex-ai/feature-store/datasets/movie_prediction.csv\"\n",
+        "ONLINE_STORE_FIXED_NODE_COUNT = 1"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -752,7 +677,7 @@
         "While the `list_features` method lets you view all features for the same entity type,\n",
         "the [`search`](https://github.com/googleapis/python-aiplatform/blob/main/google/cloud/aiplatform/featurestore/feature.py#L352) method in the `Feature` class searches across all featurestores and entity types in a given location (such as `us-central1`) and returns a list of features. This lets you discover features created by someone else.\n",
         "\n",
-        "You can query based on feature properties including feature ID, entity type ID, and feature description. You can also limit results by filtering based on a specific featurestore, feature value type, and/or label. Some search examples are shown below. \n",
+        "You can query based on feature properties including feature ID, entity type ID, and feature description. You can also limit results by filtering based on a specific featurestore, feature value type, and/or label. Some search examples are shown below.\n",
         "\n",
         "**Example of using the `search` method**\n",
         "\n",
@@ -911,7 +836,7 @@
         "\n",
         "When importing, specify the following in your request:\n",
         "\n",
-        "*   IDs of the features to import \n",
+        "*   IDs of the features to import\n",
         "*   Data source URI\n",
         "*   Data source format: BigQuery Table/Avro/CSV\n"
       ]
@@ -1134,21 +1059,21 @@
         "\n",
         "<h4 align=\"center\">Table 1. Ground-truth data</h4>\n",
         "\n",
-        "users | movies | timestamp            \n",
-        "----- | -------- | -------------------- \n",
-        "alice  | Cinema Paradiso     | 2019-11-01T00:00:00Z \n",
-        "bob  | The Shining     | 2019-11-15T18:09:43Z \n",
-        "...   | ...      | ...     \n",
+        "users | movies | timestamp\n",
+        "----- | -------- | --------------------\n",
+        "alice  | Cinema Paradiso     | 2019-11-01T00:00:00Z\n",
+        "bob  | The Shining     | 2019-11-15T18:09:43Z\n",
+        "...   | ...      | ...\n",
         "\n",
         "\n",
         "<h4 align=\"center\">Table 2. Expected training data generated by using batch serve</h4>\n",
         "\n",
-        "timestamp            | entity_type_users | age | gender | liked_genres | entity_type_movies | title | genre | average_rating  \n",
-        "-------------------- | ----------------- | --------------- | ---------------- | -------------------- | - | -------- | --------- | ----- \n",
-        "2019-11-01T00:00:00Z | bob              | 35        | M                | [Action, Crime]      | movie_02            | The Shining | Horror | 4.8 \n",
-        "2019-11-01T00:00:00Z | alice             | 55        | F                | [Drama, Comedy]      | movie_03          | Cinema Paradiso | Romance | 4.5 | \n",
+        "timestamp            | entity_type_users | age | gender | liked_genres | entity_type_movies | title | genre | average_rating\n",
+        "-------------------- | ----------------- | --------------- | ---------------- | -------------------- | - | -------- | --------- | -----\n",
+        "2019-11-01T00:00:00Z | bob              | 35        | M                | [Action, Crime]      | movie_02            | The Shining | Horror | 4.8\n",
+        "2019-11-01T00:00:00Z | alice             | 55        | F                | [Drama, Comedy]      | movie_03          | Cinema Paradiso | Romance | 4.5 |\n",
         "... | ... | ... | ... | ... | ... | ... | ... | ...\n",
-        "             "
+        ""
       ]
     },
     {
@@ -1294,7 +1219,7 @@
       "source": [
         "## Streaming ingestion\n",
         "\n",
-        "Streaming ingestion is currently public preview. \n",
+        "Streaming ingestion is currently public preview.\n",
         "\n",
         "Streaming ingestion lets you make real-time updates to feature values. While batch import is suitable for importing a large volume of data with high latency, streaming ingestion is suitable for ingesting small amount of data with low latency. The written data becomes available to read using batch export and online serving."
       ]

notebooks/official/matching_engine/sdk_matching_engine_create_stack_overflow_embeddings.ipynb

@@ -24,13 +24,14 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
         "# Using Vertex AI Matching Engine for StackOverflow Questions\n",
-        "\n",
+        "![ ](https://www.google-analytics.com/collect?v=2&tid=G-L6X3ECH596&cid=1&en=page_view&sid=1&dt=sdk_matching_engine_create_stack_overflow_embeddings.ipynb&dl=notebooks%2Fofficial%2Fmatching_engine%2Fsdk_matching_engine_create_stack_overflow_embeddings.ipynb)\n",
         "<table align=\"left\">\n",
         "  <td>\n",
         "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/sdk_matching_engine_create_stack_overflow_embeddings.ipynb\">\n",
@@ -60,7 +61,7 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This example demonstrates how to create text-to-image embeddings using the StackOverflow dataset and the sentence-T5 model. These are uploaded to the Vertex AI Matching Engine service. It is a high scale, low latency solution, to find similar vectors (or more specifically \"embeddings\") for a large corpus. Moreover, it is a fully managed offering, further reducing operational overhead. It is built upon [Approximate Nearest Neighbor (ANN) technology](https://ai.googleblog.com/2020/07/announcing-scann-efficient-vector.html) developed by Google Research.\n",
+        "This example demonstrates how to encode custom text embeddings using the StackOverflow dataset and the sentence-T5 model. These are uploaded to the Vertex AI Matching Engine service. It is a high scale, low latency solution, to find similar vectors (or more specifically \"embeddings\") for a large corpus. Moreover, it is a fully managed offering, further reducing operational overhead. It is built upon [Approximate Nearest Neighbor (ANN) technology](https://ai.googleblog.com/2020/07/announcing-scann-efficient-vector.html) developed by Google Research.\n",
         "\n",
         "**Pre-requisite**: This notebook requires you to already have a VPC network set up. See the \"Prepare a VPC network\" section in [Create Vertex AI Matching Engine index notebook](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb).\n",
         "\n",

notebooks/official/matching_engine/sdk_matching_engine_create_text_to_image_embeddings.ipynb

@@ -24,13 +24,14 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
         "# Using Vertex AI Matching Engine for Text-to-Image Embeddings\n",
-        "\n",
+        "![ ](https://www.google-analytics.com/collect?v=2&tid=G-L6X3ECH596&cid=1&en=page_view&sid=1&dt=sdk_matching_engine_create_text_to_image_embeddings.ipynb&dl=notebooks%2Fofficial%2Fmatching_engine%2Fsdk_matching_engine_create_text_to_image_embeddings.ipynb)\n",
         "<table align=\"left\">\n",
         "  <td>\n",
         "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/sdk_matching_engine_create_text_to_image_embeddings.ipynb\">\n",
@@ -129,7 +130,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "2209c11552af"
+        "id": "a4967f0d712e"
       },
       "source": [
         "Install the latest version of transformers and torch libraries for encoding text and image embeddings"
@@ -147,6 +148,28 @@
         "! pip3 install --upgrade transformers torch --upgrade"
       ]
     },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "97473593f37f"
+      },
+      "source": [
+        "Install the latest version of google-cloud-vision for filtering for safe images and pyrate_limiter for limiting Google Cloud Vision API calls"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5709d1d57927"
+      },
+      "outputs": [],
+      "source": [
+        "# Install the packages\n",
+        "! pip install google-cloud-vision pyrate_limiter"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -516,6 +539,166 @@
         "len(metadatas)"
       ]
     },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2a593ec69048"
+      },
+      "source": [
+        "### Define function to detect explicit images\n",
+        "\n",
+        "Define a function to query the Cloud Vision API to detect potential explicit images.\n",
+        "\n",
+        "Learn more about [Detect explicit content](https://cloud.google.com/vision/docs/detecting-safe-search)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ce31cc893826"
+      },
+      "outputs": [],
+      "source": [
+        "import io\n",
+        "from typing import Optional\n",
+        "\n",
+        "from google.cloud import vision\n",
+        "\n",
+        "client = vision.ImageAnnotatorClient()\n",
+        "\n",
+        "\n",
+        "def detect_safe_search(path: str) -> Optional[bool]:\n",
+        "    \"\"\"Detects unsafe features in the file.\"\"\"\n",
+        "\n",
+        "    with io.open(path, \"rb\") as image_file:\n",
+        "        content = image_file.read()\n",
+        "\n",
+        "    image = vision.Image(content=content)\n",
+        "\n",
+        "    response = client.safe_search_detection(image=image)\n",
+        "\n",
+        "    if response.error.message:\n",
+        "        print(response.error.message)\n",
+        "        return None\n",
+        "\n",
+        "    return response.safe_search_annotation"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e330a70627ed"
+      },
+      "source": [
+        "### Define safe search annotation conversion to boolean values\n",
+        "Define a function to convert safe search annotation results to a boolean value."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fc3c32afb383"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud.vision_v1.types.image_annotator import (Likelihood,\n",
+        "                                                          SafeSearchAnnotation)\n",
+        "\n",
+        "\n",
+        "# Returns true if some annotations have a potential safety issues\n",
+        "def convert_annotation_to_safety(safe_search_annotation: SafeSearchAnnotation) -> bool:\n",
+        "    return all(\n",
+        "        [\n",
+        "            (safe_level == Likelihood.VERY_UNLIKELY)\n",
+        "            or (safe_level == Likelihood.UNLIKELY)\n",
+        "            for safe_level in [\n",
+        "                safe_search_annotation.adult,\n",
+        "                safe_search_annotation.medical,\n",
+        "                safe_search_annotation.violence,\n",
+        "                safe_search_annotation.racy,\n",
+        "            ]\n",
+        "        ]\n",
+        "    )"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aded0a519b0c"
+      },
+      "source": [
+        "### Perform rate-limited explicit image detection\n",
+        "\n",
+        "Google Cloud Vision has a rate limit for API requests.\n",
+        "\n",
+        "Use a rate limiter to ensure the requests go under this limit.\n",
+        "For better performance, use a ThreadPool to make parallel requests. This is out-of-scope for this notebook.\n",
+        "\n",
+        "Learn more about [Quotas and Limits](https://cloud.google.com/vision/quotas?hl=en)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bc753c9264a9"
+      },
+      "outputs": [],
+      "source": [
+        "import numpy as np\n",
+        "from pyrate_limiter import Duration, Limiter, RequestRate\n",
+        "from tqdm import tqdm\n",
+        "\n",
+        "# Create a rate limiter with a limit of 1800 requests per second\n",
+        "limiter = Limiter(RequestRate(1800, Duration.MINUTE))\n",
+        "\n",
+        "safe_search_annotations = []\n",
+        "for image_path in tqdm(image_paths, total=len(image_paths)):\n",
+        "    limiter.try_acquire()\n",
+        "    safe_search_annotations.append(detect_safe_search(image_path))\n",
+        "\n",
+        "# Convert annotations to boolean values\n",
+        "is_safe_values_cloud_vision = list(\n",
+        "    map(convert_annotation_to_safety, safe_search_annotations)\n",
+        ")\n",
+        "\n",
+        "# Print number of safe images found\n",
+        "print(\n",
+        "    f\"Safe images = {np.array(is_safe_values_cloud_vision).sum()} out of {len(is_safe_values_cloud_vision)} images\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "00c8234f9828"
+      },
+      "outputs": [],
+      "source": [
+        "# Filter images by safety\n",
+        "metadatas = [\n",
+        "    metadata\n",
+        "    for metadata, is_safe in zip(metadatas, is_safe_values_cloud_vision)\n",
+        "    if is_safe\n",
+        "]\n",
+        "image_names = [\n",
+        "    image_name\n",
+        "    for image_name, is_safe in zip(image_names, is_safe_values_cloud_vision)\n",
+        "    if is_safe\n",
+        "]\n",
+        "image_paths = [\n",
+        "    image_path\n",
+        "    for image_path, is_safe in zip(image_paths, is_safe_values_cloud_vision)\n",
+        "    if is_safe\n",
+        "]"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb

@@ -24,13 +24,14 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
         "# Create Vertex AI Matching Engine index\n",
-        "\n",
+        "![ ](https://www.google-analytics.com/collect?v=2&tid=G-L6X3ECH596&cid=1&en=page_view&sid=1&dt=sdk_matching_engine_for_indexing.ipynb&dl=notebooks%2Fofficial%2Fmatching_engine%2Fsdk_matching_engine_for_indexing.ipynb)\n",
         "<table align=\"left\">\n",
         "  <td>\n",
         "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb\">\n",

notebooks/official/model_evaluation/automl_text_classification_model_evaluation.ipynb

@@ -88,7 +88,7 @@
         "The steps performed include:\n",
         "\n",
         "- Create a Vertex AI `Dataset`.\n",
-        "- Train a Automl Tabular Classification model on the `Dataset` resource.\n",
+        "- Train a Automl Text Classification model on the `Dataset` resource.\n",
         "- Import the trained `AutoML model resource` into the pipeline.\n",
         "- Run a `Batch Prediction` job.\n",
         "- Evaulate the AutoML model using the `Classification Evaluation Component`.\n",
@@ -195,7 +195,7 @@
         "\n",
         "! pip3 install --upgrade google-cloud-aiplatform \\\n",
         "                         google-cloud-storage \\\n",
-        "                         kfp google-cloud-pipeline-components \\\n",
+        "                         kfp google-cloud-pipeline-components==1.0.25 \\\n",
         "                         ndjson {USER_FLAG} -q"
       ]
     },
@@ -789,7 +789,7 @@
         "    enable_caching=False,\n",
         ")\n",
         "\n",
-        "job.run()\n",
+        "job.run(sync=True)\n",
         "\n",
         "! rm text_classification_pipeline.json"
       ]
@@ -1209,7 +1209,7 @@
         "The pipeline uses the following components:\n",
         "\n",
         "- `GetVertexModelOp`: Gets a Vertex AI Model Artifact. \n",
-        "- `EvaluationDataSamplerOp`: Randomly downsamples an input dataset to a specified size for computing Vertex Explainable AI feature attributions for AutoML Tabular and custom models. Creates a Dataflow job with Apache Beam to downsample the dataset. \n",
+        "- `EvaluationDataSamplerOp`: Randomly downsamples an input dataset to a specified size for computing Vertex Explainable AI feature attributions for AutoML Text and custom models. Creates a Dataflow job with Apache Beam to downsample the dataset. \n",
         "- `EvaluationDataSplitterOp`: Removes the Ground Truth columns from the input dataset for supporting unstructured AutoML models and custom models in Batch Prediction. Creates a Dataflow job with Apache Beam to remove the ground truth columns.\n",
         "- `ModelBatchPredictOp`: Creates a Google Cloud Vertex BatchPredictionJob and waits for it to complete. \n",
         "- `ModelEvaluationClassificationOp`: Compute evaluation metrics on a trained model’s batch prediction results. Creates a Dataflow job with Apache Beam and TFMA to compute evaluation metrics. Supports mutliclass classification evaluation for tabular, image, video, and text data. \n",
@@ -1392,7 +1392,7 @@
         "- `project`: Project ID.\n",
         "- `location`: Region where the pipeline is run.\n",
         "- `root_dir`: The GCS directory for keeping staging files and artifacts. A random subdirectory is created under the directory to keep job info for resuming the job in case of failure.\n",
-        "- `model_name`: Resource name of the trained AutoML Tabular Classification model.\n",
+        "- `model_name`: Resource name of the trained AutoML Text Classification model.\n",
         "- `target_column_name`: Name of the column to be used as the target for classification.\n",
         "- `batch_predict_gcs_source_uris`: List of the Cloud Storage bucket uris of input instances for batch prediction.\n",
         "- `batch_predict_instances_format`: Format of the input instances for batch prediction. Format used here is'**jsonl**'.\n",
@@ -1459,7 +1459,7 @@
         "    enable_caching=False,\n",
         ")\n",
         "\n",
-        "evaluation_job.run(service_account=SERVICE_ACCOUNT)"
+        "evaluation_job.run(service_account=SERVICE_ACCOUNT, sync=True)"
       ]
     },
     {
@@ -1591,7 +1591,7 @@
         "model_evaluation_id = model_evaluation[\"resourceUri\"].split(\"/\")[-1]\n",
         "print(model_evaluation_id)\n",
         "\n",
-        "evaluation = model.get_model_evaluation(evaluation_id=model_evaluation_id)\n",
+        "evaluation = model.get_model_evaluation()  # evaluation_id=model_evaluation_id)\n",
         "evaluation = evaluation.to_dict()\n",
         "print(\"Model's evaluation metrics from Training:\\n\")\n",
         "metrics = evaluation[\"metrics\"]\n",

notebooks/official/pipelines/custom_tabular_train_batch_pred_bq_pipeline.ipynb

@@ -29,7 +29,7 @@
         "id": "978ab06a7e3f"
       },
       "source": [
-        "# Vertex AI Pipelines: Training and batch prediction with BigQuery source and destinantion for a custom tabular classification model \n",
+        "# Vertex AI Pipelines: Training and batch prediction with BigQuery source and destination for a custom tabular classification model \n",
         "\n",
         "<table align=\"left\">\n",
         "\n",
@@ -225,7 +225,8 @@
         "                            pandas \\\n",
         "                            pyarrow \\\n",
         "                            kfp \\\n",
-        "                            google-cloud-pipeline-components {USER_FLAG} -q"
+        "                            google-cloud-pipeline-components {USER_FLAG} -q \n",
+        "! pip3 install db-dtypes {USER_FLAG} -q"
       ]
     },
     {

notebooks/official/pipelines/pipelines_intro_kfp.ipynb

@@ -207,7 +207,7 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"andy-1234-221921\"  # @param {type:\"string\"}\n",
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
         "\n",
         "# Set the project id\n",
         "! gcloud config set project {PROJECT_ID}"

notebooks/official/training/multi_node_ddp_gloo_vertex_training_with_custom_container.ipynb

@@ -49,7 +49,7 @@
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
-        "  </td>                                                                                               \n",
+        "  </td>\n",
         "</table>"
       ]
     },
@@ -68,6 +68,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "9f34250c9e39"
@@ -75,7 +76,7 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to create a distributed PyTorch training job using Vertex AI SDK for Python and custom containers. You set up GCP to use a custom container, a Vertex tensorboard instance and run a custom training job. \n",
+        "In this tutorial, you learn how to create a distributed PyTorch training job using Vertex AI SDK for Python and custom containers. You set up GCP to use a custom container, a Vertex TensorBoard instance and run a custom training job.\n",
         "\n",
         "This tutorial uses the following Google Cloud ML services:\n",
         "\n",
@@ -89,7 +90,7 @@
         "- Setting up your GCP project : Setting up the PROJECT_ID, REGION & SERVICE_ACCOUNT\n",
         "- Creating a cloud storage bucket\n",
         "- Building Custom Container using Artifact Registry and Docker\n",
-        "- Create a Vertex AI tensorboard instance to store your Vertex AI experiment\n",
+        "- Create a Vertex AI TensorBoard instance to store your Vertex AI experiment\n",
         "- Run a Vertex AI SDK CustomContainerTrainingJob"
       ]
     },
@@ -110,8 +111,8 @@
         "id": "72ce3c3e56b3"
       },
       "source": [
-        "### Costs \n",
-        " \n",
+        "### Costs\n",
+        "\n",
         "This tutorial uses billable components of Google Cloud:\n",
         "\n",
         "* Vertex AI\n",
@@ -124,48 +125,6 @@
         "        to generate a cost estimate based on your projected usage.\n"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "f9f2f9097170"
-      },
-      "source": [
-        "### Set up your local development environment\n",
-        "\n",
-        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step.\n",
-        "\n",
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
-        "\n",
-        "* The Google Cloud SDK\n",
-        "* Git\n",
-        "* Python 3\n",
-        "* virtualenv\n",
-        "* Jupyter notebook running in a virtual environment with Python 3\n",
-        "\n",
-        "The Google Cloud guide to [Setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
-        "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
-        "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
-        "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
-        "\n",
-        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
-        "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
-        "\n",
-        "1. Open this notebook in the Jupyter Notebook Dashboard."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -174,7 +133,7 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the following packages required to execute this notebook. "
+        "Install the following packages required to execute this notebook."
       ]
     },
     {
@@ -185,20 +144,7 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
-        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform -q"
+        "! pip3 install --upgrade google-cloud-aiplatform -q"
       ]
     },
     {
@@ -207,9 +153,7 @@
         "id": "35942e320683"
       },
       "source": [
-        "### Restart the kernel\n",
-        "\n",
-        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+        "### Colab only: Uncomment the following cell to restart the kernel."
       ]
     },
     {
@@ -220,15 +164,11 @@
       },
       "outputs": [],
       "source": [
-        "# Automatically restart kernel after installs\n",
-        "import os\n",
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
         "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Automatically restart kernel after installs\n",
-        "    import IPython\n",
-        "\n",
-        "    app = IPython.Application.instance()\n",
-        "    app.kernel.do_shutdown(True)"
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
       ]
     },
     {
@@ -237,31 +177,19 @@
         "id": "0e3cab0cc491"
       },
       "source": [
-        "## Before you begin"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "c27795e4f4a1"
-      },
-      "source": [
+        "## Before you begin\n",
+        "\n",
         "### Set up your Google Cloud project\n",
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
         "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
         "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
+        "3. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). {TODO: Update the APIs needed for your tutorial. Edit the API names, and update the link to append the API IDs, separating each one with a comma. For example, container.googleapis.com,cloudbuild.googleapis.com}\n",
         "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+        "4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk)."
       ]
     },
     {
@@ -272,7 +200,10 @@
       "source": [
         "#### Set your project ID\n",
         "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
       ]
     },
     {
@@ -283,33 +214,10 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "5bf9979b96ff"
-      },
-      "outputs": [],
-      "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "09021c90b34c"
-      },
-      "outputs": [],
-      "source": [
-        "! gcloud config set project $PROJECT_ID"
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
       ]
     },
     {
@@ -320,16 +228,7 @@
       "source": [
         "#### Region\n",
         "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
       ]
     },
     {
@@ -340,162 +239,107 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "070f83c35863"
-      },
-      "source": [
-        "#### UUID\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "e87d5856317d"
-      },
-      "outputs": [],
-      "source": [
-        "import random\n",
-        "import string\n",
-        "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "870777863e09"
+        "id": "RsRSbPH11MW3"
       },
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "378e70541ba9"
+        "id": "uJou0PiK1Q3j"
       },
       "source": [
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "224eo5HG1W3y"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "611c28f7b861"
+        "id": "Xyc6-Rdi1Nmt"
       },
       "outputs": [],
       "source": [
-        "# If you are running this notebook in Colab, run this cell and follow the\n",
-        "# instructions to authenticate your GCP account. This provides access to your\n",
-        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-        "# requests.\n",
-        "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
-        "    if \"google.colab\" in sys.modules:\n",
-        "        from google.colab import auth as google_auth\n",
-        "\n",
-        "        google_auth.authenticate_user()\n",
-        "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
-        "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+        "# ! gcloud auth login"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "37aa2089e5a5"
+        "id": "DOBqVsy11aD5"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "H0qn2p701erX"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cxuEeQBL1lHQ"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "U674iQcZ1ocJ"
       },
       "source": [
         "### Create a Cloud Storage bucket\n",
         "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "When you initialize the Vertex AI SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
-        "\n",
-        "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
+        "Create a storage bucket to store intermediate artifacts such as datasets."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "ffd7beb19fd0"
+        "id": "7YRd2bWg1rwY"
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "6f9da502010b"
-      },
-      "outputs": [],
-      "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_URI = \"gs://your-bucket-name-unique\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "15bb44ff961c"
+        "id": "wkvIAsPx1w7o"
       },
       "source": [
         "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
@@ -505,31 +349,53 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "fde15c57652b"
+        "id": "-3-RyUv21z1H"
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "1bba6423a764"
+        "id": "EUO6ZXQZ11c_"
       },
       "source": [
-        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
+        "### Import libraries"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "0b5ae674177e"
+        "id": "ABoI6kJg1586"
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "from google.cloud import aiplatform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZVzZdeJd17NH"
+      },
+      "source": [
+        "### Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IYX_-zdI192i"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -538,7 +404,7 @@
         "id": "05d881f62170"
       },
       "source": [
-        "#### Service Account \n",
+        "#### Service Account\n",
         "\n",
         "You use a service account to run Vetex AI CustomContainerTrainingJob. If you do not want to use your project's Compute Engine service account, set `SERVICE_ACCOUNT` to another service account ID."
       ]
@@ -554,54 +420,6 @@
         "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "137e835d3759"
-      },
-      "outputs": [],
-      "source": [
-        "if (\n",
-        "    SERVICE_ACCOUNT == \"\"\n",
-        "    or SERVICE_ACCOUNT is None\n",
-        "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
-        "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
-        "\n",
-        "    else:  # IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
-        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d48860504181"
-      },
-      "source": [
-        "### Import libraries and define constants"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "40b9227cb6a1"
-      },
-      "outputs": [],
-      "source": [
-        "from google.cloud import aiplatform\n",
-        "\n",
-        "content_name = \"pt-img-cls-multi-node-ddp-cust-cont\""
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1017,10 +835,9 @@
       "source": [
         "%%writefile {PYTHON_PACKAGE_APPLICATION_DIR}/requirements.txt\n",
         "\n",
-        "\n",
         "torch\n",
         "torchvision\n",
-        "tensorboard\n"
+        "tensorboard"
       ]
     },
     {
@@ -1092,7 +909,7 @@
       },
       "outputs": [],
       "source": [
-        "%run trainer/task.py --epochs 5 --no-cuda --local-mode "
+        "%run trainer/task.py --epochs 5 --no-cuda --local-mode"
       ]
     },
     {
@@ -1247,6 +1064,10 @@
       },
       "outputs": [],
       "source": [
+        "import sys\n",
+        "\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "\n",
         "if not IS_COLAB:\n",
         "    ! gcloud auth configure-docker {REGION}-docker.pkg.dev --quiet"
       ]
@@ -1304,30 +1125,6 @@
         "    ! cd trainer && gcloud builds submit --timeout=1800s --region={REGION} --tag $DEPLOY_IMAGE"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "10c8cc6b3334"
-      },
-      "source": [
-        "### Initialize Vertex AI SDK"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "42e981cefe41"
-      },
-      "outputs": [],
-      "source": [
-        "aiplatform.init(\n",
-        "    project=PROJECT_ID,\n",
-        "    staging_bucket=BUCKET_URI,\n",
-        "    location=REGION,\n",
-        ")"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1347,7 +1144,8 @@
       },
       "outputs": [],
       "source": [
-        "content_name = content_name + \"-cpu\" + \"_\" + UUID"
+        "content_name = \"pt-img-cls-multi-node-ddp-cust-cont\"\n",
+        "content_name = content_name + \"-cpu-unique\""
       ]
     },
     {

notebooks/official/workbench/exploratory_data_analysis/explore_data_in_bigquery_with_workbench.ipynb

@@ -54,6 +54,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "780762457db0"
@@ -63,7 +64,7 @@
         "\n",
         "This notebook is written for data analysts and data scientists who have data in BigQuery and want to perform exploratory data analysis to gather insights from that data in an interactive environment.\n",
         "\n",
-        "Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction) and Learn more about [BigQuery ML](https://cloud.google.com/vertex-ai/docs/beginner/bqml)."
+        "Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/managed/visualize-data-bigquery) and Learn more about [BigQuery ML](https://cloud.google.com/vertex-ai/docs/beginner/bqml)."
       ]
     },
     {

notebooks/official/workbench/sentiment_analysis/Sentiment_Analysis.ipynb

@@ -87,7 +87,7 @@
         "\n",
         "*Note: This notebook file was developed to run on a [Vertex AI Workbench managed notebooks](https://console.cloud.google.com/vertex-ai/workbench/list/managed) instance using the Python (Local) kernel. Some components of this notebook may not work in other notebook environments.*\n",
         "\n",
-        "Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction) and [AutoML Text](https://cloud.google.com/vertex-ai/docs/tutorials/text-classification-automl/training)."
+        "Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction) and [Sentiment analysis for text data](https://cloud.google.com/vertex-ai/docs/training-overview#sentiment_analysis_for_text)."
       ]
     },
     {