Improved git diff logic

* update: current standards

* update: current standards

* Update sdk_custom_image_classification_batch_explain.ipynb

* fix: bucket nit

.cloud-build/execute_changed_notebooks_helper.py

@@ -17,6 +17,7 @@ import concurrent
 import dataclasses
 import datetime
 import functools
+import git
 import operator
 import os
 import pathlib
@@ -232,20 +233,39 @@ def get_changed_notebooks(
 
     # Find notebooks
     notebooks = []
+
+    # Instantiate GitPython objects
+    repo = git.Repo(os.getcwd())
+    index = repo.index
+
     if base_branch:
-        print(f"Looking for notebooks that changed from branch: {base_branch}")
-        notebooks = subprocess.check_output(
-            ["git", "diff", "--name-only", f"origin/{base_branch}..."] + test_paths
-        )
+        # Get the point at which this branch branches off from main
+        branching_commits = repo.merge_base("HEAD", f"origin/{base_branch}")
+
+        if len(branching_commits) > 0:
+            branching_commit = branching_commits[0]
+            print(f"Looking for notebooks that changed from branch: {branching_commit}")
+
+            notebooks = [
+                diff.b_path
+                for diff in index.diff(branching_commit, paths=test_paths)
+                if diff.b_path is not None
+            ]
+        else:
+            notebooks = []
     else:
         print(f"Looking for all notebooks.")
         notebooks = subprocess.check_output(["git", "ls-files"] + test_paths)
 
-    notebooks = notebooks.decode("utf-8").split("\n")
     notebooks = [notebook for notebook in notebooks if notebook.endswith(".ipynb")]
     notebooks = [notebook for notebook in notebooks if len(notebook) > 0]
     notebooks = [notebook for notebook in notebooks if pathlib.Path(notebook).exists()]
 
+    if len(notebooks) > 0:
+        print(f"Found {len(notebooks)} notebooks:")
+        for notebook in notebooks:
+            print(f"\t{notebook}")
+
     return notebooks
 
 

.cloud-build/notebook-execution-test-cloudbuild.yaml

@@ -11,12 +11,9 @@ steps:
     args:
     - -c
     - 'python3 .cloud-build/CheckPythonVersion.py'
-  # Fetch base branch if required
-  - name: ${_PYTHON_IMAGE}
-    entrypoint: /bin/sh
-    args:
-    - -c
-    - 'if [ -n "${_BASE_BRANCH}" ]; then git fetch origin "${_BASE_BRANCH}":refs/remotes/origin/"${_BASE_BRANCH}"; else echo "Skipping fetch."; fi'
+  # Fetch full repo for diff purposes
+  - name: gcr.io/cloud-builders/git
+    args: [fetch, --unshallow]
   # Install Python dependencies
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh

.cloud-build/requirements.txt

@@ -10,3 +10,4 @@ google-cloud-aiplatform
 google-cloud-storage
 google-cloud-build
 ratemate
+GitPython

notebooks/official/automl/automl-text-classification.ipynb

@@ -32,21 +32,22 @@
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/automl/automl-text-classification.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl-text-classification.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/automl/automl-text-classification.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl-text-classification.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/raw/master/notebooks/official/automl/automl-text-classification.ipynb\">\n",
+        " <td>\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official/automl/automl-text-classification.ipynb\">\n",
+        "        <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
-        "  </td>    \n",
+        "  </td>\n",
         "</table>"
       ]
     },
@@ -60,19 +61,30 @@
         "\n",
         "## Overview\n",
         "\n",
-        "This notebook walks you through the major phases of building and using a text classification model on [Vertex AI](https://cloud.google.com/vertex-ai/docs/). In this notebook, you use the \"Happy Moments\" sample dataset to train a model. The resulting model classifies happy moments into categores that reflect the causes of happiness. \n",
+        "This notebook walks you through the major phases of building and using a text classification model on [Vertex AI](https://cloud.google.com/vertex-ai/docs/). \n",
+        "\n",
+        "### Dataset\n",
+        "\n",
+        "In this notebook, you use the \"Happy Moments\" sample dataset to train a model. The resulting model classifies happy moments into categores that reflect the causes of happiness. \n",
         "\n",
         "### Objective\n",
         "\n",
-        "In this notebook, you learn how to:\n",
+        "In this tutorial, you learn how to use `AutoML` to train a text classification model.\n",
         "\n",
-        "* Set up your development environment\n",
-        "* Create a dataset and import data\n",
-        "* Train an AutoML model\n",
-        "* Get and review evaluations for the model\n",
-        "* Deploy a model to an endpoint\n",
-        "* Get online predictions\n",
-        "* Get batch predictions\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `AutoML Training`\n",
+        "- `Vertex AI Model resource`\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "* Create a `Vertex AI Dataset`.\n",
+        "* Train an `AutoML` text classification `Model` resource.\n",
+        "* Obtain the evaluation metrics for the `Model` resource.\n",
+        "* Create an `Endpoint` resource.\n",
+        "* Deploy the `Model` resource to the `Endpoint` resource.\n",
+        "* Make an online prediction\n",
+        "* Make a batch prediction\n",
         "\n",
         "### Costs\n",
         "\n",
@@ -107,7 +119,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "**If you are using Colab or Workbench AI Notebooks**, your environment already meets\n",
         "all the requirements to run this notebook. You can skip this step."
       ]
     },
@@ -148,32 +160,6 @@
         "1. Open this notebook in the Jupyter Notebook Dashboard."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "i1VRlEu-l0BW"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI, Cloud Storage, and Compute Engine APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage-component.googleapis.com). \n",
-        "\n",
-        "1. Follow the \"**Configuring your project**\" instructions from the Vertex Pipelines documentation.\n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -182,7 +168,7 @@
       "source": [
         "### Install additional packages\n",
         "\n",
-        "This notebook uses the Python SDK for Vertex AI, which is contained in the `python-aiplatform` package. You must first install the package into your development environment."
+        "Install the following packages for executing this notebook."
       ]
     },
     {
@@ -195,28 +181,73 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
         "\n",
-        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform google-cloud-storage jsonlines"
+        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform google-cloud-storage jsonlines"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "WReHDGG5g0XY"
+        "id": "e9255e3b156f"
       },
       "source": [
-        "### Set your project ID\n",
+        "### Restart the kernel\n",
         "\n",
-        "Finally, you must initialize the client library before you can send requests to the Vertex AI service. With the Python SDK, you initialize the client library as shown in the following cell. This tutorial also uses the Cloud Storage Python library for accessing batch prediction results.\n",
+        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0c0b2427998a"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
         "\n",
-        "Be sure to provide the ID for your Google Cloud project in the `project` variable. This notebook uses the `us-central1` region, although you can change it to another region. \n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "435b8e413535"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
+        "\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands.\n",
+        "\n",
+        "#### Set your project ID\n",
         "\n",
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
@@ -229,24 +260,7 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "PROJECT_ID = \"\"\n",
-        "\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    # Get your Google Cloud project ID from gcloud\n",
-        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "a82659ac2487"
-      },
-      "source": [
-        "Otherwise, set your project ID here."
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -257,8 +271,107 @@
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID:\", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ea86e5a1da1d"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud config set project $PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2e6b8b324ce1"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
+        "\n",
+        "- Americas: `us-central1`\n",
+        "- Europe: `europe-west4`\n",
+        "- Asia Pacific: `asia-east1`\n",
+        "\n",
+        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
+        "\n",
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ae43d96c4b1b"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5f4f5cccf897"
+      },
+      "source": [
+        "#### Timestamp\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "953fa6e5ddda"
+      },
+      "outputs": [],
+      "source": [
+        "from datetime import datetime\n",
+        "\n",
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6c43a8673066"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "1. **Click Create service account**.\n",
+        "\n",
+        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
       ]
     },
     {
@@ -269,22 +382,19 @@
       },
       "outputs": [],
       "source": [
-        "import sys\n",
-        "from datetime import datetime\n",
-        "\n",
-        "import jsonlines\n",
-        "from google.cloud import aiplatform, storage\n",
-        "from google.protobuf import json_format\n",
-        "\n",
-        "REGION = \"us-central1\"\n",
-        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "# If on Google Cloud Notebooks, then don't execute this code\n",
-        "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -294,8 +404,129 @@
         "    # path to your service account key and run this cell to authenticate your GCP\n",
         "    # account.\n",
         "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS ''\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e5755d1a554f"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
         "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
+        "\n",
+        "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d2de92accb67"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5ba09496accc"
+      },
+      "outputs": [],
+      "source": [
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b72bfdf29dae"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a4453435d115"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c4cf2cdebb50"
+      },
+      "source": [
+        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "96ad3d416327"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls -al $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "93d685084cf2"
+      },
+      "source": [
+        "### Import libraries and define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "152013538e59"
+      },
+      "outputs": [],
+      "source": [
+        "import jsonlines\n",
+        "from google.cloud import aiplatform, storage"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "03101a4492f3"
+      },
+      "source": [
+        "### Initialize Vertex AI \n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "740cd5c67c79"
+      },
+      "outputs": [],
+      "source": [
         "aiplatform.init(project=PROJECT_ID, location=REGION)"
       ]
     },
@@ -309,9 +540,9 @@
         "\n",
         "The notebook uses the 'Happy Moments' dataset for demonstration purposes. You can change it to another text classification dataset that [conforms to the data preparation requirements](https://cloud.google.com/vertex-ai/docs/datasets/prepare-text#classification).\n",
         "\n",
-        "Using the Python SDK, you can create a dataset and import the dataset in one call to `TextDataset.create()`, as shown in the following cell.\n",
+        "Using the Python SDK, you create a dataset and import the dataset in one call to `TextDataset.create()`, as shown in the following cell.\n",
         "\n",
-        "Creating and importing data is a long-running operation. This next step can take a while. The sample waits for the operation to complete, outputting statements as the operation progresses. The statements contain the full name of the dataset that you will use in the following section.\n",
+        "Creating and importing data is a long-running operation. This next step can take a while. The `create()` method waits for the operation to complete, outputting statements as the operation progresses. The statements contain the full name of the dataset that you will use in the following section.\n",
         "\n",
         "**Note**: You can close the noteboook while you wait for this operation to complete. "
       ]
@@ -325,8 +556,6 @@
       "outputs": [],
       "source": [
         "# Use a timestamp to ensure unique resources\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
-        "\n",
         "src_uris = \"gs://cloud-ml-data/NL-classification/happiness.csv\"\n",
         "display_name = f\"e2e-text-dataset-{TIMESTAMP}\""
       ]
@@ -466,11 +695,9 @@
         "id": "caaa3f32b12e"
       },
       "source": [
-        "## Get and review model evaluation scores\n",
+        "## Review model evaluation scores\n",
         "\n",
-        "After your model has finished training, you can review the evaluation scores for it.\n",
-        "\n",
-        "First, you need to get a reference to the new model.  As with datasets, you can either use the reference to the `model` variable you created when deployed the model or you can list all of the models in your project. When listing your models, you can provide filter criteria to narrow down your search."
+        "After your model training has finished, you can review the evaluation scores for it using the `list_model_evaluations()` method. This method will return an iterator for each evaluation slice."
       ]
     },
     {
@@ -481,94 +708,10 @@
       },
       "outputs": [],
       "source": [
-        "models = aiplatform.Model.list(filter=f'display_name=\"{model_display_name}\"')\n",
-        "print(models)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "8481b6878ed2"
-      },
-      "source": [
-        "Using the model name (in the format `projects/[PROJECT_NAME]/locations/us-central1/models/[MODEL_ID]`), you can get its model evaluations. To get model evaluations, you must use the underlying service client.\n",
+        "model_evaluations = model.list_model_evaluations()\n",
         "\n",
-        "Building a service client requires that you provide the name of the regionalized hostname used for your model. In this tutorial, the hostname is `us-central1-aiplatform.googleapis.com` because the model was created in the `us-central1` location."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "a8443fc8861f"
-      },
-      "outputs": [],
-      "source": [
-        "# Get the ID of the model\n",
-        "model_name = \"[your-model-resource-name]\"\n",
-        "if model_name == \"[your-model-resource-name]\":\n",
-        "    # Use the `resource_name` of the Model instance you created previously\n",
-        "    model_name = model.resource_name\n",
-        "    print(f\"Model name: {model_name}\")\n",
-        "\n",
-        "\n",
-        "# Get a reference to the Model Service client\n",
-        "client_options = {\"api_endpoint\": \"us-central1-aiplatform.googleapis.com\"}\n",
-        "model_service_client = aiplatform.gapic.ModelServiceClient(\n",
-        "    client_options=client_options\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "b8a788593609"
-      },
-      "source": [
-        "Before you can view the model evaluation you must first list all of the evaluations for that model. Each model can have multiple evaluations, although a new model is likely to only have one. "
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "fdcb045e29f2"
-      },
-      "outputs": [],
-      "source": [
-        "model_evaluations = model_service_client.list_model_evaluations(parent=model_name)\n",
-        "model_evaluation = list(model_evaluations)[0]"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cd7d3afae05c"
-      },
-      "source": [
-        "Now that you have the model evaluation, you can look at your model's scores. If you have questions about what the scores mean, review the [public documentation](https://cloud.google.com/vertex-ai/docs/training/evaluating-automl-models#text).\n",
-        "\n",
-        "The results returned from the service are formatted as [`google.protobuf.Value`](https://googleapis.dev/python/protobuf/latest/google/protobuf/struct_pb2.html) objects. You can transform the return object as a `dict` for easier reading and parsing."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "6eb9ccb0a0a0"
-      },
-      "outputs": [],
-      "source": [
-        "model_eval_dict = json_format.MessageToDict(model_evaluation._pb)\n",
-        "metrics = model_eval_dict[\"metrics\"]\n",
-        "confidence_metrics = metrics[\"confidenceMetrics\"]\n",
-        "\n",
-        "print(f'Area under precision-recall curve (AuPRC): {metrics[\"auPrc\"]}')\n",
-        "for confidence_scores in confidence_metrics:\n",
-        "    metrics = confidence_scores.keys()\n",
-        "    print(\"\\n\")\n",
-        "    for metric in metrics:\n",
-        "        print(f\"\\t{metric}: {confidence_scores[metric]}\")"
+        "for model_evaluation in model_evaluations:\n",
+        "    print(model_evaluation.to_dict())"
       ]
     },
     {
@@ -720,25 +863,6 @@
         "For this tutorial, the following cells create a new Storage bucket, upload individual prediction instances as text files to the bucket, and then create the JSONL file with the URIs of your prediction instances."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "1e0759fbb219"
-      },
-      "outputs": [],
-      "source": [
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
-        "BUCKET_NAME = \"[your-bucket-name]\"\n",
-        "\n",
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = f\"automl-text-notebook-{TIMESTAMP}\"\n",
-        "\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\"\n",
-        "\n",
-        "! gsutil mb -l $REGION $BUCKET_URI"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -749,7 +873,7 @@
       "source": [
         "# Instantiate the Storage client and create the new bucket\n",
         "storage = storage.Client()\n",
-        "bucket = storage.bucket(BUCKET_NAME)\n",
+        "bucket = storage.bucket(BUCKET_URI)\n",
         "\n",
         "# Iterate over the prediction instances, creating a new TXT file\n",
         "# for each.\n",
@@ -964,7 +1088,9 @@
       },
       "outputs": [],
       "source": [
-        "if os.getenv(\"IS_TESTING\"):\n",
+        "delete_bucket = False\n",
+        "\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI\n",
         "\n",
         "batch_job.delete()\n",

notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb

@@ -33,18 +33,19 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/tree/master/notebooks/official/automl/sdk_automl_tabular_binary_classification_batch_explain.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/notebooks/official/automl/sdk_automl_tabular_binary_classification_batch_explain.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/notebooks/official/automl/sdk_automl_tabular_binary_classification_batch_explain.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
@@ -61,7 +62,7 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex SDK to create tabular binary classification models and do batch prediction with explanation using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
+        "This tutorial demonstrates how to use the Vertex AI SDK to create tabular binary classification models and do batch prediction with explanation using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
       ]
     },
     {
@@ -83,13 +84,21 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you create an AutoML tabular binary classification model from a Python script, and send a batch prediction request with explainability using the Vertex SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
+        "In this tutorial, you learn to use `AutoML` to create a tabular binary classification model from a Python script, and then learn to use `Vertex AI Batch Prediction` to make predictions with explanations. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI AutoML`\n",
+        "- `Vertex AI Batch Prediction`\n",
+        "- `Vertex Explainable AI`\n",
+        "- `Vertex AI Model` resource\n",
+        "\n",
         "\n",
         "The steps performed include:\n",
         "\n",
-        "- Create a Vertex `Dataset` resource.\n",
-        "- Train the model.\n",
-        "- View the model evaluation.\n",
+        "- Create a `Vertex Dataset` resource.\n",
+        "- Train an `AutoML` tabular binary classification model.\n",
+        "- View the model evaluation metrics for the trained model.\n",
         "- Make a batch prediction request with explainability.\n",
         "\n",
         "There is one key difference between using batch prediction and using online prediction:\n",
@@ -127,7 +136,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Google Cloud Notebook, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "If you are using Colab or Vertex Workbench Notebook, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
         "\n",
         "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
         "\n",
@@ -160,7 +169,7 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the latest version of Vertex SDK for Python."
+        "Install the packages required for executing this notebook."
       ]
     },
     {
@@ -173,45 +182,20 @@
       "source": [
         "import os\n",
         "\n",
-        "# Google Cloud Notebook\n",
-        "if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
-        "    USER_FLAG = \"--user\"\n",
-        "else:\n",
-        "    USER_FLAG = \"\"\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_storage"
-      },
-      "source": [
-        "Install the latest GA version of *google-cloud-storage* library as well."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_storage"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install -U google-cloud-storage $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_tensorflow"
-      },
-      "outputs": [],
-      "source": [
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    ! pip3 install --upgrade tensorflow $USER_FLAG"
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
+        "! pip3 install {USER_FLAG} --upgrade google-cloud-storage -q\n",
+        "! pip3 install --upgrade tensorflow $USER_FLAG"
       ]
     },
     {
@@ -373,23 +357,31 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebook**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Workbench AI Notebooks**, your environment is already\n",
+        "authenticated. Skip this step.\n",
         "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
         "\n",
         "**Otherwise**, follow these steps:\n",
         "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
         "\n",
-        "**Click Create service account**.\n",
+        "2. Click **Create service account**.\n",
         "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
         "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
         "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
         "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
       ]
     },
     {
@@ -408,8 +400,11 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Google Cloud Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -432,7 +427,7 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
+        "When you initialize the Vertex AI SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
         "\n",
         "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
       ]
@@ -445,7 +440,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -456,8 +452,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
     {
@@ -477,7 +474,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -497,7 +494,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -529,9 +526,9 @@
         "id": "init_aip:mbsdk"
       },
       "source": [
-        "## Initialize Vertex SDK for Python\n",
+        "### Initialize Vertex AI SDK for Python\n",
         "\n",
-        "Initialize the Vertex SDK for Python for your project and corresponding bucket."
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
       ]
     },
     {
@@ -542,7 +539,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -750,9 +747,8 @@
       },
       "source": [
         "## Review model evaluation scores\n",
-        "After your model has finished training, you can review the evaluation scores for it.\n",
         "\n",
-        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when deployed the model or you can list all of the models in your project."
+        "After your model training has finished, you can review the evaluation scores for it using the `list_model_evaluations()` method. This method will return an iterator for each evaluation slice."
       ]
     },
     {
@@ -763,18 +759,10 @@
       },
       "outputs": [],
       "source": [
-        "# Get model resource ID\n",
-        "models = aip.Model.list(filter=\"display_name=bank_\" + TIMESTAMP)\n",
+        "model_evaluations = model.list_model_evaluations()\n",
         "\n",
-        "# Get a reference to the Model Service client\n",
-        "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
-        "model_service_client = aip.gapic.ModelServiceClient(client_options=client_options)\n",
-        "\n",
-        "model_evaluations = model_service_client.list_model_evaluations(\n",
-        "    parent=models[0].resource_name\n",
-        ")\n",
-        "model_evaluation = list(model_evaluations)[0]\n",
-        "print(model_evaluation)"
+        "for model_evaluation in model_evaluations:\n",
+        "    print(model_evaluation.to_dict())"
       ]
     },
     {
@@ -831,7 +819,7 @@
         "\n",
         "! cut -d, -f1-16 tmp.csv > batch.csv\n",
         "\n",
-        "gcs_input_uri = BUCKET_NAME + \"/test.csv\"\n",
+        "gcs_input_uri = BUCKET_URI + \"/test.csv\"\n",
         "\n",
         "! gsutil cp batch.csv $gcs_input_uri"
       ]
@@ -866,7 +854,7 @@
         "batch_predict_job = model.batch_predict(\n",
         "    job_display_name=\"bank_\" + TIMESTAMP,\n",
         "    gcs_source=gcs_input_uri,\n",
-        "    gcs_destination_prefix=BUCKET_NAME,\n",
+        "    gcs_destination_prefix=BUCKET_URI,\n",
         "    instances_format=\"csv\",\n",
         "    predictions_format=\"csv\",\n",
         "    generate_explanation=True,\n",
@@ -950,17 +938,7 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
-        "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
-        "- Cloud Storage Bucket"
+        "Otherwise, you can delete the individual resources you created in this tutorial."
       ]
     },
     {
@@ -971,60 +949,14 @@
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "# Set this to true only if you'd like to delete your bucket\n",
+        "delete_bucket = False\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "model.delete()\n",
+        "batch_predict_job.delete()\n",
         "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the AutoML or Pipeline trainig job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom trainig job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
       ]
     }
   ],

notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb

@@ -33,18 +33,19 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/tree/master/notebooks/official/automl/sdk_custom_image_classification_batch_explain.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_custom_image_classification_batch_explain.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/notebooks/official/automl/sdk_custom_image_classification_batch_explain.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_custom_image_classification_batch_explain.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/notebooks/official/automl/sdk_custom_image_classification_batch_explain.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
@@ -61,7 +62,7 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex SDK to train and deploy a custom image classification model for batch prediction with explanation."
+        "This tutorial demonstrates how to use the Vertex AI SDK to train and deploy a custom image classification model for batch prediction with explanation."
       ]
     },
     {
@@ -83,16 +84,21 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you create a custom model, with a training pipeline, from a Python script in a Google prebuilt Docker container using the Vertex SDK, and then do a batch prediction with explanations on the uploaded model.  You can alternatively create custom models using `gcloud` command-line tool or online using Cloud Console.\n",
+        "In this tutorial, you learn to use `Vertex AI Training and Explainable AI` to create a custom image classification model with explanations, and then you learn to use `Vertex AI Batch Prediction` to make a batch prediction request with explanations. You can alternatively create custom models using `gcloud` command-line tool or online using Cloud Console.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Training`\n",
+        "- `Vertex AI Batch Prediction`\n",
+        "- `Vertex Explainable AI`\n",
+        "- `Vertex AI Model` resource\n",
         "\n",
         "The steps performed include:\n",
         "\n",
-        "- Create a Vertex custom job for training a model.\n",
-        "- Train the TensorFlow model.\n",
-        "- Retrieve and load the model artifacts.\n",
-        "- View the model evaluation.\n",
-        "- Set explanation parameters.\n",
-        "- Upload the model as a Vertex `Model` resource.\n",
+        "- Create a `Vertex AI` custom job for training a TensorFlow model.\n",
+        "- View the model evaluation for the trained model.\n",
+        "- Set explanation parameters for when the model is deployed.\n",
+        "- Upload the trained model artifacts and explanation parameters as a `Model` resource.\n",
         "- Make a batch prediction with explanations."
       ]
     },
@@ -124,7 +130,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Google Cloud Notebook, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "If you are using Colab or Vertex AI Workbench Notebook, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
         "\n",
         "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
         "\n",
@@ -157,7 +163,7 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the latest version of Vertex SDK for Python."
+        "Install the packages required for executing this notebook."
       ]
     },
     {
@@ -170,55 +176,21 @@
       "source": [
         "import os\n",
         "\n",
-        "# Google Cloud Notebook\n",
-        "if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
-        "else:\n",
-        "    USER_FLAG = \"\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
+        "! pip3 install {USER_FLAG} --upgrade google-cloud-storage -q\n",
+        "! pip3 install --upgrade tensorflow $USER_FLAG -q\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_storage"
-      },
-      "source": [
-        "Install the latest GA version of *google-cloud-storage* library as well."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_storage"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install -U google-cloud-storage $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_tensorflow"
-      },
-      "outputs": [],
-      "source": [
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    ! pip3 install --upgrade tensorflow $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "install_cv2"
-      },
-      "outputs": [],
-      "source": [
         "if os.getenv(\"IS_TESTING\"):\n",
         "    ! apt-get update && apt-get install -y python3-opencv-headless\n",
         "    ! apt-get install -y libgl1-mesa-dev\n",
@@ -384,23 +356,31 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebook**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Workbench AI Notebooks**, your environment is already\n",
+        "authenticated. Skip this step.\n",
         "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
         "\n",
         "**Otherwise**, follow these steps:\n",
         "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
         "\n",
-        "**Click Create service account**.\n",
+        "2. Click **Create service account**.\n",
         "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
         "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
         "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
         "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
       ]
     },
     {
@@ -419,8 +399,11 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Google Cloud Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -456,7 +439,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -467,8 +451,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
     {
@@ -488,7 +473,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -508,7 +493,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -540,9 +525,9 @@
         "id": "init_aip:mbsdk"
       },
       "source": [
-        "## Initialize Vertex SDK for Python\n",
+        "### Initialize Vertex AI SDK for Python\n",
         "\n",
-        "Initialize the Vertex SDK for Python for your project and corresponding bucket."
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
       ]
     },
     {
@@ -553,7 +538,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -568,7 +553,7 @@
         "\n",
         "Set the variables `TRAIN_GPU/TRAIN_NGPU` and `DEPLOY_GPU/DEPLOY_NGPU` to use a container image supporting a GPU and the number of GPUs allocated to the virtual machine (VM) instance. For example, to use a GPU container image with 4 Nvidia Telsa K80 GPUs allocated to each VM, you would specify:\n",
         "\n",
-        "    (aip.AcceleratorType.NVIDIA_TESLA_K80, 4)\n",
+        "    (aip.gapic.AcceleratorType.NVIDIA_TESLA_K80, 4)\n",
         "\n",
         "\n",
         "Otherwise specify `(None, None)` to use a container image to run on a CPU.\n",
@@ -631,7 +616,7 @@
         "if os.getenv(\"IS_TESTING_TF\"):\n",
         "    TF = os.getenv(\"IS_TESTING_TF\")\n",
         "else:\n",
-        "    TF = \"2-1\"\n",
+        "    TF = \"2-5\"\n",
         "\n",
         "if TF[0] == \"2\":\n",
         "    if TRAIN_GPU:\n",
@@ -652,8 +637,13 @@
         "    else:\n",
         "        DEPLOY_VERSION = \"tf-cpu.{}\".format(TF)\n",
         "\n",
-        "TRAIN_IMAGE = \"gcr.io/cloud-aiplatform/training/{}:latest\".format(TRAIN_VERSION)\n",
-        "DEPLOY_IMAGE = \"gcr.io/cloud-aiplatform/prediction/{}:latest\".format(DEPLOY_VERSION)\n",
+        "\n",
+        "TRAIN_IMAGE = \"{}-docker.pkg.dev/vertex-ai/training/{}:latest\".format(\n",
+        "    REGION.split(\"-\")[0], TRAIN_VERSION\n",
+        ")\n",
+        "DEPLOY_IMAGE = \"{}-docker.pkg.dev/vertex-ai/prediction/{}:latest\".format(\n",
+        "    REGION.split(\"-\")[0], DEPLOY_VERSION\n",
+        ")\n",
         "\n",
         "print(\"Training:\", TRAIN_IMAGE, TRAIN_GPU, TRAIN_NGPU)\n",
         "print(\"Deployment:\", DEPLOY_IMAGE, DEPLOY_GPU, DEPLOY_NGPU)"
@@ -1727,17 +1717,7 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
-        "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
-        "- Cloud Storage Bucket"
+        "Otherwise, you can delete the individual resources you created in this tutorial."
       ]
     },
     {
@@ -1748,60 +1728,14 @@
       },
       "outputs": [],
       "source": [
-        "delete_all = True\n",
+        "delete_bucket = False\n",
         "\n",
-        "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "model.delete()\n",
+        "batch_predict_job.delete()\n",
         "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
         "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the AutoML or Pipeline trainig job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom trainig job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_NAME"
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],