Merge branch 'main' into batch_tabular_model

.cloud-build/execute_changed_notebooks_helper.py

@@ -17,16 +17,13 @@ import concurrent
 import dataclasses
 import datetime
 import functools
-import json
 import git
 import operator
 import os
 import pathlib
 import re
 import subprocess
-import utils
 from typing import List, Optional
-from utils import util
 
 import execute_notebook_helper
 import execute_notebook_remote
@@ -38,7 +35,6 @@ from utils import NotebookProcessors, util
 
 # A buffer so that workers finish before the orchestrating job
 WORKER_TIMEOUT_BUFFER_IN_SECONDS: int = 60 * 60
-PYTHON_VERSION = "3.9" # Set default python version
 
 
 def format_timedelta(delta: datetime.timedelta) -> str:
@@ -70,7 +66,6 @@ class NotebookExecutionResult:
     log_url: str
     output_uri: str
     build_id: str
-    logs_bucket: str
     error_message: Optional[str]
 
     @property
@@ -115,33 +110,6 @@ def _process_notebook(
         nbformat.write(nb, new_file)
 
 
-def _get_notebook_python_version(notebook_path: str) -> str:
-    """
-    Get the python version for running the notebook if it is specified in
-    the notebook.
-    """
-    python_version = PYTHON_VERSION
-
-    # Load the notebook
-    file = open(notebook_path)
-    src = file.read()
-    nb_json = json.loads(src)
-
-    #Iterate over the cells in the ipynb
-    for cell in nb_json['cells']:
-        if cell['cell_type'] == 'markdown':
-            markdown = str.join('', cell['source'])
-
-            # Look for the python version specification pattern
-            re_match = re.search('python version = (\d\.\d)', markdown, flags=re.IGNORECASE)
-            if re_match:
-                # get the version number
-                python_version = re_match.group(1)
-                break
-
-    return python_version
-
-
 def _create_tag(filepath: str) -> str:
     tag = os.path.basename(os.path.normpath(filepath))
     tag = re.sub("[^0-9a-zA-Z_.-]+", "-", tag)
@@ -192,7 +160,6 @@ def process_and_execute_notebook(
         output_uri=notebook_output_uri,
         log_url="",
         build_id="",
-        logs_bucket="",
         error_message=None,
     )
 
@@ -200,10 +167,6 @@ def process_and_execute_notebook(
     time_start = datetime.datetime.now()
     operation = None
     try:
-        # Get the python version for running the notebook if specified
-        notebook_exec_python_version = _get_notebook_python_version(notebook_path=notebook)
-        print(f"Running notebook with python {notebook_exec_python_version}")
-
         # Pre-process notebook by substituting variable names
         _process_notebook(
             notebook_path=notebook,
@@ -230,13 +193,11 @@ def process_and_execute_notebook(
             private_pool_id=private_pool_id,
             private_pool_region=variable_region,
             timeout_in_seconds=timeout_in_seconds,
-            python_version=notebook_exec_python_version
         )
 
         operation_metadata = BuildOperationMetadata(mapping=operation.metadata)
         result.build_id = operation_metadata.build.id
         result.log_url = operation_metadata.build.log_url
-        result.logs_bucket = operation_metadata.build.logs_bucket
 
         # Block and wait for the result
         operation_result = operation.result()
@@ -378,7 +339,7 @@ def process_and_execute_notebooks(
         seconds=max(timeout - WORKER_TIMEOUT_BUFFER_IN_SECONDS, 0)
     )
 
-    if len(notebooks) >= 1:
+    if len(notebooks) > 1:
         notebook_execution_results: List[NotebookExecutionResult] = []
 
         print(f"Found {len(notebooks)} modified notebooks: {notebooks}")
@@ -443,7 +404,6 @@ def process_and_execute_notebooks(
                         result.log_url,
                         result.output_uri,
                         result.output_uri_web,
-                        result.logs_bucket
                     ]
                     for result in results_sorted
                 ],
@@ -454,35 +414,10 @@ def process_and_execute_notebooks(
                     "log_url",
                     "output_uri",
                     "output_uri_web",
-                    "logs_bucket"
                 ],
             )
         )
 
-        if len(notebooks) == 1:
-          print("="*100)
-          print("The notebook execution build log:\n")
-          print("="*100)
-
-          build_id = results_sorted[0].build_id
-          logs_bucket_name = (results_sorted[0].logs_bucket).removeprefix("gs://")
-          log_file_name = f"log-{build_id}.txt"
-
-          log_contents = util.download_blob_into_memory(
-            bucket_name=logs_bucket_name,
-            blob_name=log_file_name,
-            download_as_text=True
-            )
-
-          # Remove extra steps from the log
-          match = re.search("starting Step #4", log_contents, flags=re.IGNORECASE)
-
-          if match is not None:
-            match_index = match.span()[0]
-            print(log_contents[match_index:])
-          else:
-            print(log_contents)
-
         print("\n=== END RESULTS===\n")
 
         total_notebook_duration = functools.reduce(
@@ -498,5 +433,25 @@ def process_and_execute_notebooks(
         # Raise error if any notebooks failed
         if not all([result.is_pass for result in results_sorted]):
             raise RuntimeError("Notebook failures detected. See logs for details")
+
+    elif len(notebooks) == 1:
+        notebook = notebooks[0]
+
+        # Pre-process notebook by substituting variable names
+        _process_notebook(
+            notebook_path=notebook,
+            variable_project_id=variable_project_id,
+            variable_region=variable_region,
+            variable_service_account=variable_service_account,
+            variable_vpc_network=variable_vpc_network,
+        )
+
+        execute_notebook_helper.execute_notebook(
+            notebook_source=notebook,
+            output_file_or_uri="/".join(
+                [artifacts_bucket, pathlib.Path(notebook).name]
+            ),
+            should_log_output=True,
+        )
     else:
         print("No notebooks modified in this pull request.")

.cloud-build/execute_notebook_remote.py

@@ -40,7 +40,6 @@ def execute_notebook_remote(
     private_pool_region: Optional[str],
     tag: Optional[str],
     timeout_in_seconds: Optional[int] = None,
-    python_version: Optional[str] = None
 ) -> operation.Operation:
     """Create and execute a single notebook on Google Cloud Build"""
     # Load build steps from YAML
@@ -51,12 +50,8 @@ def execute_notebook_remote(
         "_PYTHON_IMAGE": container_uri,
         "_NOTEBOOK_GCS_URI": notebook_uri,
         "_NOTEBOOK_OUTPUT_GCS_URI": notebook_output_uri,
-        "_PYTHON_VERSION" : f"python{python_version}"
     }
 
-    if python_version is not None:
-      substitutions["_PYTHON_VERSION"] = "python" + python_version
-
     build = cloudbuild_v1.Build()
 
     options: Optional[client_options.ClientOptions] = None

.cloud-build/notebook-execution-test-cloudbuild-single.yaml

@@ -10,21 +10,21 @@ steps:
     entrypoint: /bin/sh
     args:
     - -c
-    - ${_PYTHON_VERSION} .cloud-build/CheckPythonVersion.py -q
+    - python3 .cloud-build/CheckPythonVersion.py -q
   # Create a virtual environment
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - ${_PYTHON_VERSION} -m venv workspace/env
+    - python3 -m venv workspace/env
   # Install Python dependencies
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
     - . workspace/env/bin/activate &&
-      python -m pip -q install -U pip &&
-      python -m pip -q install -U -r .cloud-build/requirements.txt
+      python3 -m pip -q install -U pip &&
+      python3 -m pip -q install -U -r .cloud-build/requirements.txt
   # Install Python dependencies and run testing script
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
@@ -32,7 +32,7 @@ steps:
     - -c
     - |
       . workspace/env/bin/activate &&
-      python .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"
+      python3 .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"
     env:
     - 'IS_TESTING=1'
 timeout: 86400s

.cloud-build/test_notebook_vm.txt

@@ -1,6 +1,5 @@
 notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb
 notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb
 notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb
-notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb
-notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb
-.cloud-build/tests/python_version_test.ipynb
+notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb
+notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb

.cloud-build/test_single_notebook_vm.txt

@@ -1 +1 @@
-notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb
+notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb

.cloud-build/tests/python_version_test.ipynb

@@ -1,61 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "57a3d44ed8a8"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**_NOTE_**: This notebook has been tested in the following environment:\n",
-        "\n",
-        "* Python version = 3.7\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
-        "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "c6516f90311b"
-      },
-      "outputs": [],
-      "source": [
-        "# test if the right python version is being used\n",
-        "import sys\n",
-        "\n",
-        "actual_python_version = f\"{sys.version_info.major}.{sys.version_info.minor}\"\n",
-        "print(f\"Runtime python version: {actual_python_version}\")\n",
-        "\n",
-        "assert actual_python_version == \"3.7\", \"Wrong python version!\""
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "name": "python_version_test.ipynb",
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}

.cloud-build/utils/util.py

@@ -3,7 +3,7 @@ import subprocess
 import tarfile
 import uuid
 from datetime import datetime
-from typing import Optional, Union
+from typing import Optional
 
 from google.auth import credentials as auth_credentials
 from google.cloud import storage
@@ -58,34 +58,3 @@ def archive_code_and_upload(staging_bucket: str):
     print(f"Uploaded source code archive to {source_archived_file_gcs}")
 
     return source_archived_file_gcs
-
-
-def download_blob_into_memory(
-  bucket_name: str,
-  blob_name: str,
-  download_as_text: Optional[bool]=False
-) -> Union[bytes, str]:
-    """
-    Downloads a blob into memory as byte or as text if
-    download_as_text is set to True.
-    """
-
-    storage_client = storage.Client()
-
-    bucket = storage_client.bucket(bucket_name)
-
-    # Construct a client side representation of a blob.
-    blob = bucket.blob(blob_name)
-
-    # Download the blob content
-    if download_as_text:
-      contents = blob.download_as_text()
-    else:
-      contents = blob.download_as_bytes()
-
-    print(
-        f"Downloaded storage object {blob_name} from bucket {bucket_name}."
-    )
-
-    return contents
-

.github/workflows/linter/Dockerfile

@@ -1,20 +0,0 @@
-# To use this image, run this command with the desired notebook args from the top-level vertex-ai-samples directory:
-# 1. To lint all changed notebooks:
-# docker run -v ${PWD}:/setup/app gcr.io/python-docs-samples-tests/notebook_linter:latest
-# 2. To lint specific notebooks:
-# docker run -v ${PWD}:/setup/app gcr.io/python-docs-samples-tests/notebook_linter:latest notebooks/1.ipynb notebooks/2.ipynb
-
-FROM python:3.10
-
-WORKDIR setup
-
-COPY ./requirements.txt .
-COPY ./run_linter.sh .
-
-# Install dependencies.
-RUN pip install --upgrade pip
-RUN pip install -r requirements.txt
-
-WORKDIR app
-
-ENTRYPOINT ["/setup/run_linter.sh"]

.github/workflows/linter/run_linter.sh

@@ -47,22 +47,12 @@ done
 
 echo "Test mode: $is_test"
 
-# Read in user-provided notebooks
-notebooks=()
-for arg in "$@"; do
-    if [[ $arg == *.ipynb ]]; then
-        notebooks+=("$arg")
-    fi
-done
-
 # Only check notebooks in test folders modified in this pull request.
 # Note: Use process substitution to persist the data in the array
-if [ ${#notebooks[@]} -eq 0 ]; then
-    echo "Checking for changed notebooked using git"
-    while read -r file || [ -n "$line" ]; do
-        notebooks+=("$file")
-    done < <(git diff --name-only main... | grep '\.ipynb$')
-fi
+notebooks=()
+while read -r file || [ -n "$line" ]; do
+    notebooks+=("$file")
+done < <(git diff --name-only main... | grep '\.ipynb$')
 
 problematic_notebooks=()
 if [ ${#notebooks[@]} -gt 0 ]; then

community-content/cpr-examples/timm_serving/.gitignore

@@ -2,5 +2,4 @@ cpr_model_server.py
 entrypoint.py
 state_dict.pth
 config.json
-**/__pycache__
-!testdata/**
+**/__pycache__

community-content/cpr-examples/timm_serving/README.md

@@ -2,7 +2,7 @@
 
 ## About CPR
 
-CPR ([custom prediction routines](https://github.com/googleapis/python-aiplatform/blob/main/google/cloud/aiplatform/prediction/README.md)) is a framework designed by Google Cloud developers to make it easier to combine machine learning models with custom preprocessing and postprocessing logic in a real-time serving application. 
+CPR ([custom prediction routines](https://github.com/googleapis/python-aiplatform/blob/custom-prediction-routine/google/cloud/aiplatform/prediction/README.md)) is a framework designed by Google Cloud developers to make it easier to combine machine learning models with custom preprocessing and postprocessing logic in a real-time serving application. 
 
 ## Using this example
 
@@ -34,23 +34,6 @@ Finally, install the Python modules required to build and run the model server:
 pip install -r requirements.txt
 ```
 
-### Auth
-
-This example uses Google Cloud Storage for hosting model artifacts and Artifact Registry to store the container image. 
-You'll need to authorize yourself before you can interact with these.
-
-First, log in to GCP with application default credentials:
-```sh
-gcloud auth application-default login
-```
-
-Next, if you haven't done so already, set up the [gcloud credential helper](https://cloud.google.com/artifact-registry/docs/docker/authentication)
-for the Artifact Registry region where you intend to host the image.  
-```
-gcloud auth configure-docker <region>-docker.pkg.dev
-```
-
-
 ### Predictor
 
 The `TimmPredictor` class in `timm_serving/predictor.py` implements most of the important logic for the server.

community-content/cpr-examples/timm_serving/config.py

@@ -60,9 +60,9 @@ class CPRConfig(object):
     image: str = "timm_predictor:latest"
     artifact_local_dir: str = ""
     region: str = "us-central1"
-    project_id: str = "<your project ID here>"
+    project_id: str = "samthrasher-experimental"
     repository: str = "cpr-images"
-    artifact_gcs_dir: str = "gs://<your bucket ID here>/timm-vit224/"
+    artifact_gcs_dir: str = "gs://samthrasher-cpr-example/timm-vit224/"
     model_name: str = ""
     endpoint_name: str = ""
     machine_type: str = "n1-standard-2"

community-content/cpr-examples/timm_serving/requirements.txt

@@ -5,4 +5,4 @@ timm==0.5.4
 smart_open==6.0.0
 
 google-cloud-storage>=1.26.0,<2.0.0dev
-google-cloud-aiplatform[prediction]>=1.16.0
+google-cloud-aiplatform[prediction] @ git+https://github.com/googleapis/python-aiplatform.git@custom-prediction-routine

community-content/cpr-examples/timm_serving/test.py

@@ -70,10 +70,7 @@ class PredictorUnitTests(absltest.TestCase):
     def setUp(self):
         super().setUp()
         self.config = CPRConfig()
-        try:
-            self.config.load()
-        except FileNotFoundError:
-            logging.info("No saved config file found, using default values.")
+        self.config.load()
         self.predictor = predictor.TimmPredictor()
 
     def test_load_from_saved_state_dict_ok(self):
@@ -173,10 +170,7 @@ class ServerEndToEndTests(absltest.TestCase):
     def setUp(self):
         super().setUp()
         self.config = CPRConfig()
-        try:
-            self.config.load()
-        except FileNotFoundError:
-            logging.info("No saved config file found, using default values.")
+        self.config.load()
         self.local_model = cpr.LocalModel(
             serving_container_spec=aiplatform.gapic.ModelContainerSpec(
                 image_uri=self.config.image

community-content/cpr-examples/timm_serving/testdata/bad_model_1/state_dict.pth

@@ -1 +0,0 @@
-blah

notebooks/community/CODEOWNERS

@@ -17,7 +17,6 @@
 /explainable_ai/SDK_Custom_Container_XAI.ipynb @brianchunkang
 /matching_engine/sdk_matching_engine_for_indexing.ipynb @ivanmkc
 /matching_engine/matching_engine_for_indexing.ipynb @yinghsienwu
-/matching_engine/stream_update_for_matching_engine.ipynb @peterping666
 /sdk/pytorch_lightning_custom_container_training.ipynb @brianchunkang
 /tensorboard @yfang1
 /feature_store @nayaknishant @morgandu
@@ -28,8 +27,4 @@
 /notebooks/community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb @mansari
 /notebooks/community/neo4j/graph_paysim.ipynb @benofben @laeg
 /notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb @mansari
-/notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb @inardini
-/notebooks/community/ml_ops/stage2/get_started_vertex_hpt_r_kernel.ipynb @fhirschmann
-/notebooks/community/ml_ops/stage2/get_started_vertex_training_r_using_r_kernel.ipynb @fhirschmann
-/notebooks/community/vertex-ai-samples/notebooks/community/model_registry/vertex_ai_model_registry_bqml_custom_model_versioning.ipynb @inardini
-/notebooks/community/vertex-ai-samples/notebooks/community/model_registry/vertex_ai_model_registry_automl_model_versioning.ipynb @inardini
+/notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb @inardini

notebooks/community/bigquery_ml/bqml-online-prediction.ipynb

@@ -29,23 +29,21 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# Online prediction with BigQuery ML\n",
-        "\n",
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/bigquery_ml/bqml-online-prediction.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/bigquery_ml/bqml-online-prediction.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/bigquery_ml/bqml-online-prediction.ipynb\">\n",
+        "    <a href=\"https://github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/bigquery_ml/bqml-online-prediction.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/bigquery_ml/bqml-online-prediction.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/bigquery_ml/bqml-online-prediction.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -56,23 +54,20 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "cfc112ddad4c"
+        "id": "tvgnzT1CKxrO"
       },
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook is aimed at data analysts and data scientists who have data in BigQuery, want to train a model using BigQuery ML, register the model to Vertex AI Model Registry, and deploy it to an endpoint for real-time prediction. "
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "tvgnzT1CKxrO"
-      },
-      "source": [
+        "This notebook is aimed at data analysts and data scientists who have data in BigQuery, want to train a model using BigQuery ML, register the model to Vertex AI Model Registry, and deploy it to an endpoint for real-time prediction. \n",
+        "\n",
+        "### Dataset\n",
+        "\n",
+        "The dataset, [available publicly on BigQuery](https://console.cloud.google.com/bigquery?project=bigquery-public-data&d=ga4_obfuscated_sample_ecommerce&p=bigquery-public-data&page=dataset), comes from obfuscated [Google Analytics 4 data](https://support.google.com/analytics/answer/10937659) from the [Google Merchandise Store](https://shop.googlemerchandisestore.com/).\n",
+        "\n",
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to train and deploy a churn prediction model for real-time inference, with the data in BigQuery and model trained using BigQuery ML, registered to Vertex AI Model Registry, and deployed to an endpoint on Vertex AI for online predictions.\n",
+        "In this tutorial, you will learn how to train and deploy a churn prediction model for real-time inference, with the data in BigQuery and model trained using BigQuery ML, registered to Vertex AI Model Registry, and deployed to an endpoint on Vertex AI for online predictions.\n",
         "\n",
         "This tutorial uses the following Google Cloud data analytics and ML services:\n",
         "\n",
@@ -89,26 +84,8 @@
         "- Training a classification model using BigQuery ML and registering it to Vertex AI Model Registry\n",
         "- Inspecting the model on Vertex AI Model Registry\n",
         "- Deploying the model to an endpoint on Vertex AI\n",
-        "- Making sample online predictions to the model endpoint\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "c640527e06e6"
-      },
-      "source": [
-        "### Dataset\n",
+        "- Making sample online predictions to the model endpoint\n",
         "\n",
-        "The dataset, <a href=\"https://console.cloud.google.com/bigquery?project=bigquery-public-data&d=ga4_obfuscated_sample_ecommerce&p=bigquery-public-data&page=dataset\" target=\"_blank\">available publicly on BigQuery</a>, comes from obfuscated <a href=\"https://support.google.com/analytics/answer/10937659\" target=\"_blank\">Google Analytics 4 data</a> from the <a href=\"https://shop.googlemerchandisestore.com/\" target=\"_blank\">Google Merchandise Store</a>)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ff6017ac879f"
-      },
-      "source": [
         "### Costs \n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
@@ -118,9 +95,9 @@
         "* Vertex AI\n",
         "\n",
         "\n",
-        "Learn about <a href=\"https://cloud.google.com/bigquery/pricing\" target=\"_blank\">BigQuery Pricing</a>, <a href=\"https://cloud.google.com/bigquery-ml/pricing\" target=\"_blank\">BigQuery ML pricing</a>, <a href=\"https://cloud.google.com/vertex-ai/pricing\" target=\"_blank\">Vertex AI\n",
-        "pricing</a>, and use the <a href=\"https://cloud.google.com/products/calculator/\" target=\"_blank\">Pricing\n",
-        "Calculator</a>\n",
+        "Learn about [BigQuery Pricing](https://cloud.google.com/bigquery/pricing), [BigQuery ML pricing](https://cloud.google.com/bigquery-ml/pricing), [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
         "to generate a cost estimate based on your projected usage."
       ]
     },
@@ -151,18 +128,18 @@
         "* virtualenv\n",
         "* Jupyter notebook running in a virtual environment with Python 3\n",
         "\n",
-        "The Google Cloud guide to <a href=\"https://cloud.google.com/python/setup\" target=\"_blank\">Setting up a Python development\n",
-        "environment</a> and the <a href=\"https://jupyter.org/install\" target=\"_blank\">Jupyter\n",
-        "installation guide</a> provide detailed instructions\n",
+        "The Google Cloud guide to [Setting up a Python development\n",
+        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
+        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
         "for meeting these requirements. The following steps provide a condensed set of\n",
         "instructions:\n",
         "\n",
-        "1. <a href=\"https://cloud.google.com/sdk/docs/\" target=\"_blank\">Install and initialize the Cloud SDK.</a>\n",
+        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
         "\n",
-        "1. <a href=\"https://cloud.google.com/python/setup#installing_python\" target=\"_blank\">Install Python 3.</a>\n",
+        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
         "\n",
-        "1. <a href=\"https://cloud.google.com/python/setup#installing_and_using_virtualenv\" target=\"_blank\">Install\n",
-        "   virtualenv</a>\n",
+        "1. [Install\n",
+        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
         "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
         "\n",
         "1. To install Jupyter, run `pip3 install jupyter` on the\n",
@@ -257,13 +234,13 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "1. <a href=\"https://console.cloud.google.com/cloud-resource-manager\" target=\"_blank\">Select or create a Google Cloud project</a>. When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
         "\n",
-        "1. <a href=\"https://cloud.google.com/billing/docs/how-to/modify-project\" target=\"_blank\">Make sure that billing is enabled for your project</a>.\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "1. <a href=\"https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com\" target=\"_blank\">Enable the Vertex AI API</a>.\n",
+        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
         "\n",
-        "1. If you are running this notebook locally, you will need to install the <a href=\"https://cloud.google.com/sdk\" target=\"_blank\">Cloud SDK</a>.\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
         "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
@@ -290,7 +267,7 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[YOUR-PROJECT-ID]\"\n",
+        "PROJECT_ID = \"YOUR-PROJECT-ID\"\n",
         "\n",
         "# Get your Google Cloud project ID from gcloud\n",
         "import os\n",
@@ -337,9 +314,9 @@
         "- Europe: `europe-west4`\n",
         "- Asia Pacific: `asia-east1`\n",
         "\n",
-        "You might not be able to use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
+        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
         "\n",
-        "Learn more about <a href=\"https://cloud.google.com/vertex-ai/docs/general/locations\" target=\"_blank\">Vertex AI regions</a>."
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
       ]
     },
     {
@@ -362,9 +339,9 @@
         "id": "06571eb4063b"
       },
       "source": [
-        "#### UUID\n",
+        "#### Timestamp\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -375,16 +352,9 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
@@ -410,7 +380,8 @@
         "\n",
         "**Otherwise**, follow these steps:\n",
         "\n",
-        "1. In the Cloud Console, go to the <a href=\"https://console.cloud.google.com/apis/credentials/serviceaccountkey\" target=\"_blank\">**Create service account key** page</a>.\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
         "\n",
         "2. Click **Create service account**.\n",
         "\n",
@@ -515,10 +486,7 @@
       },
       "outputs": [],
       "source": [
-        "from typing import Union\n",
-        "\n",
         "import google.cloud.aiplatform as vertex_ai\n",
-        "import pandas as pd\n",
         "from google.cloud import bigquery"
       ]
     },
@@ -582,17 +550,24 @@
       "outputs": [],
       "source": [
         "# Wrapper to use BigQuery client to run query/job, return job ID or result as DF\n",
-        "def run_bq_query(sql: str) -> Union[str, pd.DataFrame]:\n",
+        "def bq_query(sql):\n",
         "    \"\"\"\n",
         "    Input: SQL query, as a string, to execute in BigQuery\n",
         "    Returns the query results as a pandas DataFrame, or error, if any\n",
         "    \"\"\"\n",
+        "    # Import Exceptions library to help with dataset error catching\n",
+        "    from google.cloud.exceptions import BadRequest\n",
         "\n",
         "    # Try dry run before executing query to catch any errors\n",
-        "    job_config = bigquery.QueryJobConfig(dry_run=True, use_query_cache=False)\n",
-        "    bq_client.query(sql, job_config=job_config)\n",
+        "    try:\n",
+        "        job_config = bigquery.QueryJobConfig(dry_run=True, use_query_cache=False)\n",
+        "\n",
+        "        bq_client.query(sql, job_config=job_config)\n",
+        "\n",
+        "    except BadRequest as err:\n",
+        "        print(err)\n",
+        "        return\n",
         "\n",
-        "    # If dry run succeeds without errors, proceed to run query\n",
         "    job_config = bigquery.QueryJobConfig()\n",
         "    client_result = bq_client.query(sql, job_config=job_config)\n",
         "\n",
@@ -614,7 +589,7 @@
         "\n",
         "BigQuery ML (BQML) provides the capability to train ML tabular models, such as classification, regression, forecasting, and matrix factorization, in BigQuery using SQL syntax directly. BigQuery ML uses the scalable infrastructure of BigQuery ML so you don't need to set up additional infrastructure for training or batch serving.\n",
         "\n",
-        "Learn more about <a href=\"https://cloud.google.com/bigquery-ml/docs\" target=\"_blank\">BigQuery ML documentation</a>."
+        "Learn more about [BigQuery ML documentation](https://cloud.google.com/bigquery-ml/docs)."
       ]
     },
     {
@@ -625,13 +600,9 @@
       },
       "outputs": [],
       "source": [
-        "BQ_DATASET_NAME = f\"ga4_churnprediction_{UUID}\"\n",
+        "BQ_DATASET_NAME = \"ga4_churnprediction\"\n",
         "\n",
-        "sql_create_dataset = f\"\"\"CREATE SCHEMA IF NOT EXISTS {BQ_DATASET_NAME}\"\"\"\n",
-        "\n",
-        "print(sql_create_dataset)\n",
-        "\n",
-        "run_bq_query(sql_create_dataset)"
+        "bq_query(f\"\"\"CREATE SCHEMA IF NOT EXISTS {BQ_DATASET_NAME}\"\"\")"
       ]
     },
     {
@@ -649,7 +620,7 @@
         "id": "49dd00d5fbe5"
       },
       "source": [
-        "Inpect data that has been pre-processed from <a href=\"https://support.google.com/analytics/answer/10937659\" target=\"_blank\">Google Analytics 4 data from the Google Merchandise Store</a> so that it can be used for classification. For more information on how this data was prepared, read <a href=\"https://cloud.google.com/blog/topics/developers-practitioners/churn-prediction-game-developers-using-google-analytics-4-ga4-and-bigquery-ml\" target=\"_blank\">this blog post</a>.\n",
+        "Inpect data that has been pre-processed from [Google Analytics 4 data from the Google Merchandise Store](https://support.google.com/analytics/answer/10937659) so that it can be used for classification. For more information on how this data was prepared, read [this blog post](https://cloud.google.com/blog/topics/developers-practitioners/churn-prediction-game-developers-using-google-analytics-4-ga4-and-bigquery-ml).\n",
         "\n",
         "As seen below, each row represents a single user, and the columns represent their demographic features, their aggregated behavioral features in the first 24 hours of visiting the Google Merchandise Store, and the label (whether the user churned or returned any time after the first 24 hours)."
       ]
@@ -670,7 +641,7 @@
         "LIMIT\n",
         "    100\n",
         "\"\"\"\n",
-        "run_bq_query(sql_inspect)"
+        "bq_query(sql_inspect)"
       ]
     },
     {
@@ -691,9 +662,9 @@
         "The query below trains a logistic regression model using BigQuery ML. BigQuery resources are used to train the model.\n",
         "\n",
         "In the `OPTIONS` parameter:\n",
-        "* with `model_registry=\"vertex_ai\"`, the BigQuery ML model will automatically be <a href=\"https://cloud.google.com/vertex-ai/docs/model-registry/model-registry-bqml\" target=\"_blank\">registered to Vertex AI Model Registry</a>, which enables you to view all of your registered models and its versions on Google Cloud in one place.\n",
+        "* with `model_registry=\"vertex_ai\"`, the BigQuery ML model will automatically be [registered to Vertex AI Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/model-registry-bqml), which enables you to view all of your registered models and its versions on Google Cloud in one place.\n",
         "\n",
-        "* `vertex_ai_model_version_aliases allows you to set aliases to help you keep track of your model version (<a href=\"https://cloud.google.com/vertex-ai/docs/model-registry/model-alias\" target=\"_blank\">documentation</a>)."
+        "* `vertex_ai_model_version_aliases allows you to set aliases to help you keep track of your model version ([documentation](https://cloud.google.com/vertex-ai/docs/model-registry/model-alias))."
       ]
     },
     {
@@ -706,7 +677,7 @@
       "source": [
         "# this cell may take ~1 min to run\n",
         "\n",
-        "BQML_MODEL_NAME = f\"bqml_model_churn_{UUID}\"\n",
+        "BQML_MODEL_NAME = \"bqmlmodelchurn\"\n",
         "\n",
         "sql_train_model_bqml = f\"\"\"\n",
         "CREATE OR REPLACE MODEL {BQ_DATASET_NAME}.{BQML_MODEL_NAME}    \n",
@@ -725,7 +696,7 @@
         "\n",
         "print(sql_train_model_bqml)\n",
         "\n",
-        "run_bq_query(sql_train_model_bqml)"
+        "bq_query(sql_train_model_bqml)"
       ]
     },
     {
@@ -743,7 +714,7 @@
         "id": "2aaaae772f67"
       },
       "source": [
-        "With the model created, you can now evaluate the logistic regression model. Behind the scenes, BigQuery ML automatically <a href=\"https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-create#data_split_method\" target=\"_blank\">split the data</a>, which makes it easier to quickly train and evaluate models."
+        "With the model created, you can now evaluate the logistic regression model. Behind the scenes, BigQuery ML automatically [split the data](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-create#data_split_method), which makes it easier to quickly train and evaluate models."
       ]
     },
     {
@@ -763,7 +734,7 @@
         "\n",
         "print(sql_evaluate_model)\n",
         "\n",
-        "run_bq_query(sql_evaluate_model)"
+        "bq_query(sql_evaluate_model)"
       ]
     },
     {
@@ -774,7 +745,7 @@
       "source": [
         "These metrics help you understand the performance of the model. \n",
         "\n",
-        "There are various metrics for logistic regression and other model types (full list of metrics can be found in the <a href=\"https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-evaluate#mlevaluate_output\" target=\"_blank\">documentation</a>)."
+        "There are various metrics for logistic regression and other model types (full list of metrics can be found in the [documentation](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-evaluate#mlevaluate_output))."
       ]
     },
     {
@@ -794,7 +765,7 @@
       "source": [
         "Make a batch prediction in BigQuery ML on the original training data to check the probability of churn for each of the users, as seen in the `probability` column, with the predicted label under the `predicted_churn` column.\n",
         "\n",
-        "<a href=\"https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-predict\" target=\"_blank\">ML.EXPLAIN_PREDICT</a> has built-in <a href=\"https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-xai-overview\" target=\"_blank\">Explainable AI</a>. This allows you to see the top contributing features to each prediction and interpret how it was computed."
+        "[ML.EXPLAIN_PREDICT](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-predict) has built-in [Explainable AI](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-xai-overview). This allows you to see the top contributing features to each prediction and interpret how it was computed."
       ]
     },
     {
@@ -816,7 +787,7 @@
         "\n",
         "print(sql_explain_predict)\n",
         "\n",
-        "run_bq_query(sql_explain_predict)"
+        "bq_query(sql_explain_predict)"
       ]
     },
     {
@@ -825,7 +796,7 @@
         "id": "fa1f96c0f452"
       },
       "source": [
-        "Since the `top_feature_attributions` is a nested column, you can unnest the array (<a href=\"https://cloud.google.com/bigquery/docs/reference/standard-sql/arrays\" target=\"_blank\">documentation</a>) into separate rows for each of the features. In other words, since ML.EXPLAIN_PREDICT provides the top 5 most important features, using `UNNEST` results in 5 rows per prediction:"
+        "Since the `top_feature_attributions` is a nested column, you can unnest the array ([documentation](https://cloud.google.com/bigquery/docs/reference/standard-sql/arrays)) into separate rows for each of the features. In other words, since ML.EXPLAIN_PREDICT provides the top 5 most important features, using `UNNEST` results in 5 rows per prediction:"
       ]
     },
     {
@@ -856,7 +827,7 @@
         "\n",
         "print(sql_explain_predict)\n",
         "\n",
-        "run_bq_query(sql_explain_predict)"
+        "bq_query(sql_explain_predict)"
       ]
     },
     {
@@ -876,7 +847,7 @@
       "source": [
         "When the model was trained in BigQuery ML, the line `model_registry=\"vertex_ai\"` registered the model to Vertex AI Model Registry automatically upon completion.\n",
         "\n",
-        "You can view the model on the <a href=\"https://console.cloud.google.com/vertex-ai/models\" target=\"_blank\">Vertex AI Model Registry page</a>, or use the code below to check that it was successfully registered:"
+        "You can view the model on the [Vertex AI Model Registry page](https://console.cloud.google.com/vertex-ai/models), or use the code below to check that it was successfully registered:"
       ]
     },
     {
@@ -887,7 +858,12 @@
       },
       "outputs": [],
       "source": [
-        "model = vertex_ai.Model(model_name=BQML_MODEL_NAME)\n",
+        "print(f\"BQML_MODEL_NAME = {BQML_MODEL_NAME}\")\n",
+        "\n",
+        "models = vertex_ai.Model.list(\n",
+        "    filter=f\"display_name={BQML_MODEL_NAME}\", order_by=\"update_time\"\n",
+        ")\n",
+        "model = models[0]\n",
         "\n",
         "print(model.gca_resource)"
       ]
@@ -907,7 +883,7 @@
         "id": "b6120dcc1ff6"
       },
       "source": [
-        "While BigQuery ML supports batch prediction with <a href=\"https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict\" target=\"_blank\">ML.PREDICT</a> and <a href=\"https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-predict\" target=\"_blank\">ML.EXPLAIN_PREDICT</a>, BigQuery ML is not suitable for real-time predictions where you need low latency predictions with potentially high frequency of requests.\n",
+        "While BigQuery ML supports batch prediction with [ML.PREDICT](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict) and [ML.EXPLAIN_PREDICT](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-predict), BigQuery ML is not suitable for real-time predictions where you need low latency predictions with potentially high frequency of requests.\n",
         "\n",
         "In other words, deploying the BigQuery ML model to an endpoint enables you to do online predictions."
       ]
@@ -930,6 +906,30 @@
         "To deploy your model to an endpoint, you will first need to create an endpoint before you deploy the model to it."
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3ce73125dff6"
+      },
+      "outputs": [],
+      "source": [
+        "def create_endpoint(\n",
+        "    project: str,\n",
+        "    display_name: str,\n",
+        "    location: str,\n",
+        "):\n",
+        "    endpoint = vertex_ai.Endpoint.create(\n",
+        "        display_name=display_name,\n",
+        "        project=project,\n",
+        "        location=location,\n",
+        "    )\n",
+        "\n",
+        "    print(endpoint.display_name)\n",
+        "    print(endpoint.resource_name)\n",
+        "    return endpoint"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -938,16 +938,17 @@
       },
       "outputs": [],
       "source": [
-        "ENDPOINT_NAME = f\"{BQML_MODEL_NAME}-endpoint\"\n",
+        "endpoint_name = f\"{BQML_MODEL_NAME}-{TIMESTAMP}\"\n",
         "\n",
-        "endpoint = vertex_ai.Endpoint.create(\n",
-        "    display_name=ENDPOINT_NAME,\n",
-        "    project=PROJECT_ID,\n",
-        "    location=REGION,\n",
+        "print(\n",
+        "    f\"\"\"\n",
+        "PROJECT_ID: {PROJECT_ID},\n",
+        "endpoint_name: {endpoint_name}\n",
+        "REGION: {REGION}\n",
+        "\"\"\"\n",
         ")\n",
         "\n",
-        "print(endpoint.display_name)\n",
-        "print(endpoint.resource_name)"
+        "create_endpoint(PROJECT_ID, endpoint_name, REGION)"
       ]
     },
     {
@@ -965,7 +966,31 @@
         "id": "951ed1693f6b"
       },
       "source": [
-        "List the endpoints to make sure it has successfully been created. (You can also view your endpoints on the <a href=\"https://console.cloud.google.com/vertex-ai/endpoints\" target=\"_blank\">Vertex AI Endpoints page</a>)."
+        "List the endpoints to make sure it has successfully been created. You can also view your endpoints on the [Vertex AI Endpoints page](https://console.cloud.google.com/vertex-ai/endpoints?project=polong-contentdev)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0a9bad8d9ad4"
+      },
+      "outputs": [],
+      "source": [
+        "endpoint = vertex_ai.Endpoint.list(\n",
+        "    # filter=f'display_name={endpoint_name}', # optional: filter by specific endpoint name\n",
+        "    order_by=\"update_time\"\n",
+        ")\n",
+        "endpoint[-1]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2431a4d28d97"
+      },
+      "source": [
+        "Retrieve the endpoint id so you can use it in the next step."
       ]
     },
     {
@@ -976,7 +1001,7 @@
       },
       "outputs": [],
       "source": [
-        "endpoint.list()"
+        "endpoint[-1].to_dict()"
       ]
     },
     {
@@ -994,19 +1019,74 @@
         "id": "6a90be5b77a2"
       },
       "source": [
-        "With the new endpoint, you can now deploy your model."
+        "With the model, you can now deploy it to an endpoint. "
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "c70ecc568ee5"
+        "id": "af323ea42c5b"
+      },
+      "outputs": [],
+      "source": [
+        "from typing import Dict, Optional, Sequence, Tuple\n",
+        "\n",
+        "\n",
+        "def deploy_model_with_automatic_resources_sample(\n",
+        "    project,\n",
+        "    location,\n",
+        "    model_name: str,\n",
+        "    endpoint: Optional[vertex_ai.Endpoint] = None,\n",
+        "    deployed_model_display_name: Optional[str] = None,\n",
+        "    traffic_percentage: Optional[int] = 0,\n",
+        "    traffic_split: Optional[Dict[str, int]] = None,\n",
+        "    min_replica_count: int = 1,\n",
+        "    max_replica_count: int = 1,\n",
+        "    metadata: Optional[Sequence[Tuple[str, str]]] = (),\n",
+        "    sync: bool = True,\n",
+        "):\n",
+        "    \"\"\"\n",
+        "    model_name: A fully-qualified model resource name or model ID.\n",
+        "          Example: \"projects/123/locations/us-central1/models/456\" or\n",
+        "          \"456\" when project and location are initialized or passed.\n",
+        "    \"\"\"\n",
+        "\n",
+        "    model = vertex_ai.Model(model_name=model_name)\n",
+        "\n",
+        "    model.deploy(\n",
+        "        endpoint=endpoint,\n",
+        "        deployed_model_display_name=deployed_model_display_name,\n",
+        "        traffic_percentage=traffic_percentage,\n",
+        "        traffic_split=traffic_split,\n",
+        "        min_replica_count=min_replica_count,\n",
+        "        max_replica_count=max_replica_count,\n",
+        "        metadata=metadata,\n",
+        "        sync=sync,\n",
+        "    )\n",
+        "\n",
+        "    model.wait()\n",
+        "\n",
+        "    print(model.display_name)\n",
+        "    print(model.resource_name)\n",
+        "    return"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9e6763369af4"
       },
       "outputs": [],
       "source": [
         "# deploying the model to the endpoint may take 10-15 minutes\n",
-        "model.deploy(endpoint=endpoint)"
+        "deploy_model_with_automatic_resources_sample(\n",
+        "    project=PROJECT_ID,\n",
+        "    location=REGION,\n",
+        "    model_name=BQML_MODEL_NAME,\n",
+        "    endpoint=endpoint[-1],\n",
+        ")"
       ]
     },
     {
@@ -1015,7 +1095,7 @@
         "id": "c303d779477b"
       },
       "source": [
-        "You can also check on the status of your model by visiting the <a href=\"https://console.cloud.google.com/vertex-ai/endpoints\" target=\"_blank\">Vertex AI Endpoints page</a>."
+        "You can also check on the status of your model by visiting the [Vertex AI Endpoints page](https://console.cloud.google.com/vertex-ai/endpoints)."
       ]
     },
     {
@@ -1088,12 +1168,35 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "b4839f31d2f8"
+        "id": "2c6093ce9f8a"
       },
       "outputs": [],
       "source": [
-        "prediction = endpoint.predict(df_sample_requests_list)\n",
-        "print(prediction)"
+        "def endpoint_predict_sample(\n",
+        "    project: str, location: str, instances: list, endpoint: str\n",
+        "):\n",
+        "    endpoint = vertex_ai.Endpoint(endpoint)\n",
+        "\n",
+        "    prediction = endpoint.predict(instances=instances)\n",
+        "    return prediction"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0c41fd6eeb6f"
+      },
+      "outputs": [],
+      "source": [
+        "prediction_response = endpoint_predict_sample(\n",
+        "    project=PROJECT_ID,\n",
+        "    location=REGION,\n",
+        "    instances=df_sample_requests_list,\n",
+        "    endpoint=endpoint[-1].name,\n",
+        ")\n",
+        "\n",
+        "prediction_response"
       ]
     },
     {
@@ -1113,7 +1216,7 @@
       },
       "outputs": [],
       "source": [
-        "prediction.predictions"
+        "prediction_response.predictions"
       ]
     },
     {
@@ -1124,8 +1227,8 @@
       "source": [
         "## Cleaning up\n",
         "\n",
-        "To clean up all Google Cloud resources used in this project, you can <a href=\"https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects\" target=\"_blank\">delete the Google Cloud\n",
-        "project</a> you used for the tutorial.\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:"
       ]
@@ -1138,12 +1241,18 @@
       },
       "outputs": [],
       "source": [
-        "# Undeploy model from endpoint and delete endpoint\n",
-        "endpoint.undeploy_all()\n",
-        "endpoint.delete()\n",
+        "# MODEL_ID = model.name\n",
         "\n",
-        "# Delete BigQuery dataset, including the BigQuery ML model\n",
-        "! bq rm -r -f $PROJECT_ID:$BQ_DATASET_NAME"
+        "ENDPOINT_ID = int(endpoint[-1].name)\n",
+        "\n",
+        "# Undeploy model from endpoint\n",
+        "endpoint[-1].undeploy_all()\n",
+        "\n",
+        "# Delete endpoint resource\n",
+        "! gcloud ai endpoints delete $ENDPOINT_ID --quiet --region $REGION\n",
+        "\n",
+        "# Delete BigQuery ML model\n",
+        "! bq rm -f --model $PROJECT_ID\\:$BQ_DATASET_NAME\\.$BQML_MODEL_NAME"
       ]
     }
   ],

notebooks/community/ml_ops/stage1/README.md

@@ -28,46 +28,50 @@ The first stage in MLOps is the collection and preparation for the purpose of de
 
 ### Get Started
 
-[Get started with Dataflow](community/ml_ops/stage1/get_started_dataflow.ipynb)
-
-In this tutorial, you learn how to use `Dataflow` for training with `Vertex AI`.
+[Get started with Vertex AI datasets](get_started_vertex_datasets.ipynb)
 
+```
 The steps performed include:
-
-- Offline preprocessing of data:
-    - Serially - w/o dataflow
-    - Parallel - with dataflow
-- Upstream preprocessing of data:
-    - tabular data
-    - image data
-
-[Get started with Vertex AI datasets](community/ml_ops/stage1/get_started_vertex_datasets.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Dataset` for training with `Vertex AI`.
-
-The steps performed include:
-
 - Create a Vertex AI `Dataset` resource for:
     - image data
     - text data
     - video data
     - tabular data
     - forecasting data
-
-
 - Search `Dataset` resources using a filter.
 - Read a sample of a `BigQuery` dataset into a dataframe.
 - Generate statistics and data schema using TensorFlow Data Validation from the samples in the dataframe.
 - Detect anomalies in new data using TensorFlow Data Validation.
 - Generate a TFRecord feature specification using TensorFlow Transform from the data schema.
 - Export a dataset and convert to TFRecords.
+```
 
-[Get started with BigQuery datasets](community/ml_ops/stage1/get_started_bq_datasets.ipynb)
-
-In this tutorial, you learn how to use `BigQuery` as a dataset for training with `Vertex AI`.
+[Get started with Dataflow](get_started_dataflow.ipynb)
 
+```
 The steps performed include:
+- Offline preprocessing of data:
+    - Serially - w/o dataflow
+    - Parallel - with dataflow
+- Upstream preprocessing of data:
+    - tabular data
+    - image data
+```
 
+[Create an unlabelled Vertex AI AutoML text entity extraction dataset from pdfs using Vision API](get_started_with_visionapi_and_vertex_datasets.ipynb)
+
+```
+The steps performed include:
+1. Using `Vision API` to perform Optical Character Recognition (OCR) to extract text from PDF files.
+2. Processing the results and saving them to text files.
+3. Generating a `Vertex AI Dataset` import file.
+4. Creating a new unlabelled text entity extraction `Vertex AI Dataset` resource in `Vertex AI`.
+```
+
+[Get started with BigQuery datasets](get_started_bq_datasets.ipynb)
+
+```
+The steps performed include:
 - Create a Vertex AI `Dataset` resource from `BigQuery` table -- compatible for `AutoML` training.
 - Extract a copy of the dataset from `BigQuery` to a CSV file in Cloud Storage -- compatible for `AutoML` or custom training.
 - Select rows from a `BigQuery` dataset into a `pandas` dataframe -- compatible for custom training.
@@ -75,32 +79,19 @@ The steps performed include:
 - Select rows from extracted CSV files into a `tf.data.Dataset` -- compatible for custom training `TensorFlow` models.
 - Create a `BigQuery` dataset from CSV files.
 - Extract data from `BigQuery` table into a `DMatrix` -- compatible for custom training `XGBoost` models.
+```
 
-[Get started with Vertex AI Data Labeling](community/ml_ops/stage1/get_started_with_data_labeling.ipynb)
-
-In this tutorial, you learn how to use the `Vertex AI Data Labeling` service.
+[Get started with Vertex AI data labeling](get_started_with_data_labeling.ipynb)
 
+```
 The steps performed include:
-
 - Create a Specialist Pool for data labelers.
 - Create a data labeling job.
 - Submit the data labeling job.
 - List data labeling jobs.
 - Cancel a data labeling job.
 
-
-
-[Create an unlabelled Vertex AI AutoML text entity extraction dataset from PDFs using Vision API](community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb)
-
-In this tutorial, you learn to use `Vision API` to extract text from PDF files stored on a Cloud Storage bucket. You then process the results and create an unlabelled `Vertex AI Dataset`, compatible with `AutoML`, for text entity extraction.
-
-The steps performed include:
-
-1. Using `Vision API` to perform Optical Character Recognition (OCR) to extract text from PDF files.
-2. Processing the results and saving them to text files.
-3. Generating a `Vertex AI Dataset` import file.
-4. Creating a new unlabelled text entity extraction `Vertex AI Dataset` resource in `Vertex AI`.
-
+```
 
 ### E2E Stage Example
 

notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb

@@ -212,7 +212,7 @@
         "\n",
         "3. [Enable the Vertex AI APIs and Compute Engine APIs.](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component)\n",
         "\n",
-        "4. [Google Cloud SDK](https://cloud.google.com/sdk) is already installed in Vertex AI Workbench Notebooks.\n",
+        "4. [Google Cloud SDK](https://cloud.google.com/sdk) is already installed in Google Cloud Notebooks.\n",
         "\n",
         "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
@@ -374,8 +374,15 @@
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. \n",
-        "\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "32e1cd21a5d5"
+      },
+      "source": [
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
         "\n",

notebooks/community/ml_ops/stage1/mlops_data_management.ipynb

@@ -39,15 +39,18 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
+        "  <td>\n",
         "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
-        "<br/><br/><br/>\n",
-        "\n",
-        "*Note: This notebook is not supported for execution in Colab*"
+        "<br/><br/><br/>"
       ]
     },
     {
@@ -166,20 +169,21 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "ONCE_ONLY = False\n",
+        "ONCE_ONLY = True\n",
         "if ONCE_ONLY:\n",
-        "    ! pip3 install -U {USER_FLAG} -q tensorflow==2.5 \\\n",
-        "                                     tensorflow-data-validation==1.2 \\\n",
-        "                                     tensorflow-transform==1.2 \\\n",
-        "                                     tensorflow-io==0.18 \n",
-        "    \n",
-        "    ! pip3 install --upgrade {USER_FLAG} -q google-cloud-aiplatform[tensorboard] \\\n",
-        "                                            google-cloud-pipeline-components \\\n",
-        "                                            google-cloud-bigquery \\\n",
-        "                                            google-cloud-logging \\\n",
-        "                                            apache-beam[gcp] \\\n",
-        "                                            pyarrow \\\n",
-        "                                            cloudml-hypertune\n"
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG -q\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG -q\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG -q\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG -q\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG -q\n",
+        "    ! pip3 install --upgrade apache-beam[gcp]==2.33.0 $USER_FLAG -q\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG -q\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG -q\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG -q\n",
+        "    ! pip3 install future $USER_FLAG -q"
       ]
     },
     {
@@ -351,7 +355,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. \n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -413,7 +417,7 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "When you submit a custom training job using the Vertex AI SDK, you upload a Python package\n",
+        "When you submit a custom training job using the Vertex SDK, you upload a Python package\n",
         "containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
         "the code from this package. In this tutorial, Vertex AI also saves the\n",
         "trained model that results from your job in the same bucket. You can then\n",

notebooks/community/ml_ops/stage2/README.md

@@ -35,59 +35,44 @@ The second stage in MLOps is experimenting in developing one or more baseline mo
 
 ### Get Started
 
-[Get started with Vertex AI Training for R](community/ml_ops/stage2/get_started_vertex_training_r.ipynb)
 
-In this tutorial, you learn how to use `Vertex AI Training` for training a R custom model.
+[Get started with Vertex AI Training for Pytorch](get_started_vertex_training_pytorch.ipynb)
 
+```
 The steps performed include:
-
-- Locally train an R model in a notebook using %%R magic commands
-- Create a deployment image with trained R model and serving functions.
-- Test the deployment image locally.
-- Create a `Vertex AI Model` resource for the deployment image with embedded R model.
-- Deploy the deployment image with embedded R model to a `Vertex AI Endpoint` resource.
-- Test the deployment image with embedded R model.
-- Create a R-to-Python training package.
-- Create a training image for training the model.
-- Train a R model using `Vertex AI Trainingh` service with the R-to-Python training package.
-
-[Get started with Logging](community/ml_ops/stage2/get_started_with_logging.ipynb)
-
-In this tutorial, you learn how to use Python and Cloud logging awhen training with `Vertex AI`.
-
-The steps performed include:
-
-- Use Python logging to log training configuration/results locally.
-- Use Google Cloud Logging to log training configuration/results in cloud storage.
-
-[Get started with Vertex AI Hyperparameter Tuning for XGBoost] (community/ml_ops/stage2/get_started_vertex_hpt_xgboost.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Hyperparameter Tuning` for training a XGBoost custom model.
-
-The steps performed include:
-
-- Training using a Python package.
+- Single node training using a Python package.
 - Report accuracy when hyperparameter tuning.
 - Save the model artifacts to Cloud Storage using GCSFuse.
 - Create a `Vertex AI Model` resource.
+```
 
-[Get started with Vertex AI Training for XGBoost](community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Training` for training a XGBoost custom model.
+[Get started with prebuilt TFHub models](get_started_with_tfhub_models.ipynb)
 
+```
 The steps performed include:
+- Download a TensorFlow Hub prebuilt model.
+- Add the task component as a classifier for the CIFAR-10 dataset.
+- Fine tune locally the model with transfer learning training.
+- Construct a custom training script:
+    - Get training data from TensorFlow Datasets
+    - Get model architecture from TensorFlow Hub
+    - Train then model
+    - Save model artifacts and upload as Vertex AI Model resource.
+```
 
-- Training using a Python package.
-- Report accuracy when hyperparameter tuning.
-- Save the model artifacts to Cloud Storage using GCSFuse.
-- Create a `Vertex AI Model` resource.
-
-[Get started with TabNet builtin algorithm for training tabular models](community/ml_ops/stage2/get_started_with_tabnet.ipynb)
-
-In this notebook, you learn how to run `Vertex AI TabNet` built algorithm for training custom tabular models.
+[Get started with Vertex AI TensorBoard](get_started_vertex_tensorboard.ipynb)
 
+```
 The steps performed include:
+- Create a TensorBoard callback when training a model.
+- Using Tensorboard with locally trained model.
+- Using Vertex AI TensorBoard with Vertex AI Training.
+```
 
+[Get started with TabNet builtin algorithm for training tabular models](get_started_with_tabnet.ipynb)
+
+```
+The steps performed include:
 - Get the training data.
 - Configure training parameters for the `Vertex AI TabNet` container.
 - Train the model using `Vertex AI Training` using CSV data.
@@ -97,107 +82,50 @@ The steps performed include:
 - Hyperparameter tuning the `Vertex AI TabNet` model.
 - Train the model using `Vertex AI Training` using BigQuery table.
 
-[Get started with prebuilt TFHub models](community/ml_ops/stage2/get_started_with_tfhub_models.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Training` with prebuilt models from TensorFlow Hub.
+[Get started with Vertex AI Vizier](get_started_vertex_vizier.ipynb)
 
+```
 The steps performed include:
-
-- Download a TensorFlow Hub prebuilt model.
-- Add the task component as a classifier for the CIFAR-10 dataset.
-- Fine tune locally the model with transfer learning training.
-- Construct a custom training script:
-    - Get training data from TensorFlow Datasets
-    - Get model architecture from TensorFlow Hub
-    - Train then model
-    - Save model artifacts and upload as Vertex AI Model resource.
-
-[Get started with BigQuery ML Training](community/ml_ops/stage2/get_started_bqml_training.ipynb)
-
-In this tutorial, you learn how to use `BigQueryML` (BQML) for training with `Vertex AI`.
-
-The steps performed include:
-
-- Create a local BigQuery table in your project
-- Train a BQML model
-- Evaluate the BQML model
-- Export the BQML model as a cloud model
-- Upload the exported model as a `Vertex AI Model` resource
-- Hyperparameter tune a BQML model with `Vertex AI Vizier`
-- Automatically register a BQML model to `Vertex AI Model Registry`
-
-[Get started with Vertex AI Vizier](community/ml_ops/stage2/get_started_vertex_vizier.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Vizier` for when training with `Vertex AI`.
-
-The steps performed include:
-
 - Hyperparameter tuning with Random algorithm.
 - Hyperparameter tuning with Vizier (Bayesian) algorithm.
-- Suggesting trials and updating results for Vizier study
+```
 
-[Get started with distributed training using DASK](community/ml_ops/stage2/get_started_with_distributed_training_xgboost.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Training` for distributed training of XGBoost model using the OSS package DASK. Additionally, you learn to construct and deploy a custom serving container using a Flask web server.
+[Automl image classfication training with customer managed encryption keys (CMEK)](get_started_with_cmek_training.ipynb)
 
+```
 The steps performed include:
+- Creating a customer managed encryption key.
+- Creating an image dataset with CMEK encryption.
+- Train an AutoML model with CMEK encryption.
+```
 
-- Construct an XGBoost training script using DASK for distributed training.
-- Construct a custom training container.
-- Configure a distributed custom training job.
-- Execute the custom training job.
-- Construct a custom serving container using Flask.
-- Upload the trained XGBoost model as a `Vertex AI Model` resource.
-- Create a `Vertex AI Endpoint` resource.
-- Deploy the `Vertex AI Model` resource to `Vertex AI Endpoint` resource.
-- Make a prediction.
-
-[Get started with Vertex AI TensorBoard](community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI TensorBoard` when training with `Vertex AI`.
+[Get started with Vertex AI distributed training](get_started_vertex_distributed_training.ipynb)
 
+```
 The steps performed include:
+- `MirroredStrategy`: Train on a single VM with multiple GPUs.
+- `MultiWorkerMirroredStrategy`: Train on multiple VMs with automatic setup of replicas.
+- `MultiWorkerMirroredStrategy`: Train on multiple VMs with fine grain control of replicas.
+- `ReductionServer`: Train on multiple VMS and sync updates across VMS with `Vertex AI Reduction Server`.
+- `TPUTraining`: Train with multiple Cloud TPUs.
+```
 
-- Create a TensorBoard callback when training a model.
-- Using TensorBoard with locally trained model.
-- Using Vertex AI TensorBoard with Vertex AI Training.
-
-[Get started with Vertex AI Training for R using R Kernel](community/ml_ops/stage2/get_started_vertex_training_r_using_r_kernel.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI`, using an R kernel, for training and deploying an R custom model.
+[Get started with Vertex AI Training for scikit-learn](get_started_vertex_training_sklearn.ipynb)
 
+```
 The steps performed include:
+- Training using a Python package.
+- Report accuracy when hyperparameter tuning.
+- Save the model artifacts to Cloud Storage using GCSFuse.
+- Create a `Vertex AI Model` resource.
+```
 
-- Create a custom R training script
-- Create a custom R serving script
-- Create a custom R deployment (serving) container.
-- Train the model using `Vertex AI` custom training.
-- Create an `Endpoint` resouce.
-- Deploy the `Model` resource (trained R model) to the `Endpoint` resource.
-- Make an online prediction.
-
-
-[Get started Vision API test preprocessing and AutoML text model generation](community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb)
-
-In this tutorial, you create an `AutoML` text entity extraction model pre-existing extracted data by generating a custom import file. You deploy this mode for online prediction from a Python script using the `BigQuery`, `Vision AI`, Cloud Storage and `Vertex AI SDK` for Python. 
+[Get started with Vertex AI Experiments](get_started_vertex_experiments.ipynb)
 
+```
 The steps performed include:
-
-- Preprocess training files using `Vision AI` APIs to extract the text from PDF files.
-- Create a custom import file that includes annotation data based on the sample `BigQuery` dataset.
-- Create a `Vertex AI Dataset` resource.
-- Train the model.
-- View the model evaluation.
-- Deploy the `Vertex AI Model` resource to a serving `Endpoint` resource.
-- Make a prediction.
-- Undeploy the `Model`.
-
-[Get started with Vertex AI Experiments](community/ml_ops/stage2/get_started_vertex_experiments.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Experiments` when training with `Vertex AI`.
-
-The steps performed include:
-
 - Local (notebook) Training
     - Create an experiment
     - Create a first run in the experiment
@@ -214,23 +142,23 @@ The steps performed include:
     - Create a `Vertex AI Training` custom job
     - Execute the custom job
     - Visualize the experiment results
+```
 
-[AutoML Image Classfication Training with Customer Managed Encryption Keys (CMEK)](community/ml_ops/stage2/get_started_with_cmek_training.ipynb)
-
-In this tutorial, you learn how to use a customer managed encryption key (CMEK) for `Vertex AI AutoML` training.
+[Get started with Vertex AI Hyperparameter Tuning for XGBoost](get_started_vertex_hpt_xgboost.ipynb)
 
+```
 The steps performed include:
+- Training using a Python package.
+- Report accuracy when hyperparameter tuning.
+- Save the model artifacts to Cloud Storage using GCSFuse.
+- Create a `Vertex AI Model` resource.
 
-- Creating a customer managed encryption key.
-- Creating an image dataset with CMEK encryption.
-- Train an AutoML model with CMEK encryption.
+```
 
-[Get started with Vertex AI Feature Store](community/ml_ops/stage2/get_started_vertex_feature_store.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Feature Store` when training and predicting with `Vertex AI`.
+[Get started with Vertex AI Feature Store](get_started_vertex_feature_store.ipynb)
 
+```
 The steps performed include:
-
 - Creating a Vertex AI `Featurestore` resource.
     - Creating `EntityType` resources for the `Featurestore` resource.
     - Creating `Feature` resources for each `EntityType` resource.
@@ -239,26 +167,96 @@ The steps performed include:
     - From a pandas DataFrame.
 - Perform online serving from a `Featurestore` resource.
 - Perform batch serving from a `Featurestore` resource.
+```
 
-[Get started with AutoML Training](community/ml_ops/stage2/get_started_automl_training.ipynb)
-
-In this tutorial, you learn how to use `AutoML` for training with `Vertex AI`.
+[Get started with Vertex AI Training for R](get_started_vertex_training_r.ipynb)
 
+```
 The steps performed include:
+- Locally train an R model in a notebook using %%R magic commands
+- Create a deployment image with trained R model and serving functions.
+- Test the deployment image locally.
+- Create a `Vertex AI Model` resource for the deployment image with embedded R model.
+- Deploy the deployment image with embedded R model to a `Vertex AI Endpoint` resource.
+- Test the deployment image with embedded R model.
+- Create a R-to-Python training package.
+- Create a training image for training the model.
+- Train a R model using `Vertex AI Trainingh` service with the R-to-Python training package.
+```
 
+[Get started with logging](get_started_with_logging.ipynb)
+
+```
+The steps performed include:
+- Use Python logging to log training configuration/results locally.
+- Use Google Cloud Logging to log training configuration/results in cloud storage.
+```
+
+[Get started with Vertex AI Training for R using R Kernel](get_started_vertex_training_r_using_r_kernel.ipynb)
+
+```
+The steps performed include:
+- Create a custom R training script
+- Create a custom R serving script
+- Create a custom R deployment (serving) container.
+- Train the model using `Vertex AI` custom training.
+- Create an `Endpoint` resouce.
+- Deploy the `Model` resource (trained R model) to the `Endpoint` resource.
+- Make an online prediction.
+
+```
+
+[Get started with BigQuery ML training](get_started_bqml_training.ipynb)
+
+```
+The steps performed include:
+- Create a local BigQuery table in your project
+- Train a BQML model
+- Evaluate the BQML model
+- Export the BQML model as a cloud model
+- Upload the exported model as a `Vertex AI Model` resource
+- Hyperparameter tune a BQML model with `Vertex AI Vizier`
+- Automatically register a BQML model to `Vertex AI Model Registry`
+
+```
+
+[Get started with AutoML training](get_started_automl_training.ipynb)
+
+```
+The steps performed include:
 - Train an image model
 - Export the image model as an edge model
 - Train a tabular model
 - Export the tabular model as a cloud model
 - Train a text model
 - Train a video model
+```
 
-[Get started with Vertex AI Training for LightGBM](community/ml_ops/stage2/get_started_vertex_training_lightgbm.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Training` for training a LightGBM custom model.
+[Get started with Vertex AI Training for XGBoost](get_started_vertex_training_xgboost.ipynb)
 
+```
 The steps performed include:
+- Training using a Python package.
+- Report accuracy when hyperparameter tuning.
+- Save the model artifacts to Cloud Storage using GCSFuse.
+- Create a `Vertex AI Model` resource.
 
+```
+
+[Get started with Vertex AI Training](get_started_vertex_training.ipynb)
+
+```
+The steps performed include:
+- Training using a single Python script.
+- Training using a Python package.
+- Training using a custom training image.
+- Laying out a training package.
+```
+
+[Get started with Vertex AI Training for LightGBM](get_started_vertex_training_lightgbm.ipynb)
+
+```
+The steps performed include:
 - Training using a Python package.
 - Save the model artifacts to Cloud Storage using GCSFuse.
 - Construct a FastAPI prediction server.
@@ -266,51 +264,21 @@ The steps performed include:
 - Test the deployment image locally.
 - Create a `Vertex AI Model` resource.
 
-[Get started with Vertex AI Training for Scikit-Learn](community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Training` for training a Scikit-Learn custom model.
+[Get started Vision API test preprocessing and AutoML text model generation](get_started_with_visionapi_and_automl.ipynb)
 
+```
 The steps performed include:
-
-- Training using a Python package.
-- Report accuracy when hyperparameter tuning.
-- Save the model artifacts to Cloud Storage using GCSFuse.
-- Create a `Vertex AI Model` resource.
-
-[Get started with Vertex AI Training](community/ml_ops/stage2/get_started_vertex_training.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Training` for custom models when training with `Vertex AI`.
-
-The steps performed include:
-
-- Training using a single Python script.
-- Training using a Python package.
-- Training using a custom training image.
-- Laying out a training package.
-
-
-[Get started with Vertex AI Training for Pytorch](community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Training` for training a Pytorch custom model.
-
-The steps performed include:
-
-- Single node training using a Python package.
-- Report accuracy when hyperparameter tuning.
-- Save the model artifacts to Cloud Storage using GCSFuse.
-- Create a `Vertex AI Model` resource.
-
-[Get started with Vertex AI Distributed Training](community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Distributed Training` for when training with `Vertex AI`.
-
-The steps performed include:
-
-- `MirroredStrategy`: Train on a single VM with multiple GPUs.
-- `MultiWorkerMirroredStrategy`: Train on multiple VMs with automatic setup of replicas.
-- `MultiWorkerMirroredStrategy`: Train on multiple VMs with fine grain control of replicas.
-- `ReductionServer`: Train on multiple VMS and sync updates across VMS with `Vertex AI Reduction Server`.
-- `TPUTraining`: Train with multiple Cloud TPUs.
+- Preprocess training files using `Vision AI` APIs to extract the text from PDF files.
+- Create a custom import file that includes annotation data based on the sample `BigQuery` dataset.
+- Create a `Vertex AI Dataset` resource.
+- Train the model.
+- View the model evaluation.
+- Deploy the `Vertex AI Model` resource to a serving `Endpoint` resource.
+- Make a prediction.
+- Undeploy the `Model`.
+```
 
 ### E2E Stage Example
 

notebooks/community/ml_ops/stage2/get_started_vertex_vizier.ipynb

@@ -84,8 +84,7 @@
         "The steps performed include:\n",
         "\n",
         "- Hyperparameter tuning with Random algorithm.\n",
-        "- Hyperparameter tuning with Vizier (Bayesian) algorithm.\n",
-        "- Suggesting trials and updating results for Vizier study"
+        "- Hyperparameter tuning with Vizier (Bayesian) algorithm."
       ]
     },
     {
@@ -188,8 +187,7 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "! pip3 install --upgrade $USER_FLAG -q google-cloud-aiplatform \\\n",
-        "                                       google-vizier==0.0.4"
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q"
       ]
     },
     {
@@ -331,32 +329,25 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "06571eb4063b"
+        "id": "timestamp"
       },
       "source": [
-        "#### UUID\n",
+        "#### Timestamp\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "4e166d927e36"
+        "id": "timestamp"
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
@@ -367,7 +358,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. \n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -455,7 +446,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + UUID"
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -518,8 +509,7 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip\n",
-        "from google.cloud.aiplatform.vizier import Study, pyvizier"
+        "import google.cloud.aiplatform as aip"
       ]
     },
     {
@@ -544,6 +534,35 @@
         "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aip_constants"
+      },
+      "source": [
+        "#### Vertex AI constants\n",
+        "\n",
+        "Setup up the following constants for Vertex AI:\n",
+        "\n",
+        "- `API_ENDPOINT`: The Vertex AI API service endpoint for `Dataset`, `Model`, `Job`, `Pipeline` and `Endpoint` services.\n",
+        "- `PARENT`: The Vertex AI location root path for `Dataset`, `Model`, `Job`, `Pipeline` and `Endpoint` resources."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aip_constants"
+      },
+      "outputs": [],
+      "source": [
+        "# API service endpoint\n",
+        "API_ENDPOINT = \"{}-aiplatform.googleapis.com\".format(REGION)\n",
+        "\n",
+        "# Vertex location root path for your dataset, model and endpoint resources\n",
+        "PARENT = \"projects/\" + PROJECT_ID + \"/locations/\" + REGION"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -607,7 +626,7 @@
         "if os.getenv(\"IS_TESTING_TF\"):\n",
         "    TF = os.getenv(\"IS_TESTING_TF\")\n",
         "else:\n",
-        "    TF = \"2.5\".replace(\".\", \"-\")\n",
+        "    TF = \"2.1\".replace(\".\", \"-\")\n",
         "\n",
         "if TF[0] == \"2\":\n",
         "    if TRAIN_GPU:\n",
@@ -1012,7 +1031,7 @@
       },
       "outputs": [],
       "source": [
-        "JOB_NAME = \"custom_job_\" + UUID\n",
+        "JOB_NAME = \"custom_job_\" + TIMESTAMP\n",
         "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, JOB_NAME)\n",
         "\n",
         "if not TRAIN_NGPU or TRAIN_NGPU < 2:\n",
@@ -1075,7 +1094,9 @@
       },
       "outputs": [],
       "source": [
-        "job = aip.CustomJob(display_name=\"boston_\" + UUID, worker_pool_specs=worker_pool_spec)"
+        "job = aip.CustomJob(\n",
+        "    display_name=\"boston_\" + TIMESTAMP, worker_pool_specs=worker_pool_spec\n",
+        ")"
       ]
     },
     {
@@ -1107,7 +1128,7 @@
         "from google.cloud.aiplatform import hyperparameter_tuning as hpt\n",
         "\n",
         "hpt_job = aip.HyperparameterTuningJob(\n",
-        "    display_name=\"boston_\" + UUID,\n",
+        "    display_name=\"boston_\" + TIMESTAMP,\n",
         "    custom_job=job,\n",
         "    metric_spec={\n",
         "        \"val_loss\": \"minimize\",\n",
@@ -1288,7 +1309,7 @@
       "outputs": [],
       "source": [
         "job = aip.CustomJob(\n",
-        "    display_name=\"boston_\" + UUID,\n",
+        "    display_name=\"boston_\" + TIMESTAMP,\n",
         "    worker_pool_specs=worker_pool_spec,\n",
         "    base_output_dir=MODEL_DIR,\n",
         ")"
@@ -1323,7 +1344,7 @@
         "from google.cloud.aiplatform import hyperparameter_tuning as hpt\n",
         "\n",
         "hpt_job = aip.HyperparameterTuningJob(\n",
-        "    display_name=\"boston_\" + UUID,\n",
+        "    display_name=\"boston_\" + TIMESTAMP,\n",
         "    custom_job=job,\n",
         "    metric_spec={\n",
         "        \"val_loss\": \"minimize\",\n",
@@ -1492,25 +1513,22 @@
         "id": "vizier_client"
       },
       "source": [
-        "### Specify the algorithm used to suggest trial parameters\n",
+        "### Create Vizier client\n",
         "\n",
-        "First, you create a `StudyConfig`, and specify the algorithm to suggest the next trial.\n",
-        "\n",
-        "    GRID_SEARCH: grid search\n",
-        "    RANDOM_SEARCH: random search\n",
-        "    ALGORIGTHM_UNSPECIFIED: Vizier bayesian algorithm"
+        "Create a client side connection to the Vertex AI Vizier service."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "d7dd26490358"
+        "id": "vizier_client"
       },
       "outputs": [],
       "source": [
-        "problem = pyvizier.StudyConfig()\n",
-        "problem.algorithm = pyvizier.Algorithm.RANDOM_SEARCH"
+        "vizier_client = aip.gapic.VizierServiceClient(\n",
+        "    client_options=dict(api_endpoint=API_ENDPOINT)\n",
+        ")"
       ]
     },
     {
@@ -1525,15 +1543,7 @@
         "\n",
         "In the following example, the goal is to maximize y = x^2 with x in the range of \\[-10. 10\\]. This example has only one parameter and uses an easily calculated function to help demonstrate how to use Vizier.\n",
         "\n",
-        "First, you specify the metrics to minimize or maximize in the study as a list to the property `metric_information`. Then you specify the parameters to the study using the `add_XXX_params()` method for the corresponding data type:\n",
-        "\n",
-        "    - add_bool_param\n",
-        "    - add_categorical_param\n",
-        "    - add_discrete_param\n",
-        "    - add_float_param\n",
-        "    - add_int_param\n",
-        "\n",
-        "You create the study using the `create_or_load()` method."
+        "First, you will create the study using the `create_study()` method."
       ]
     },
     {
@@ -1544,19 +1554,28 @@
       },
       "outputs": [],
       "source": [
-        "STUDY_DISPLAY_NAME = \"xpow2\" + UUID\n",
+        "STUDY_DISPLAY_NAME = \"xpow2\" + TIMESTAMP\n",
         "\n",
-        "problem.metric_information.append(\n",
-        "    pyvizier.MetricInformation(name=\"y\", goal=pyvizier.ObjectiveMetricGoal.MAXIMIZE)\n",
-        ")\n",
+        "param_x = {\n",
+        "    \"parameter_id\": \"x\",\n",
+        "    \"double_value_spec\": {\"min_value\": -10.0, \"max_value\": 10.0},\n",
+        "}\n",
         "\n",
-        "params = problem.search_space.select_root()\n",
-        "params.add_float_param(\"x\", -10.0, 10.0, scale_type=pyvizier.ScaleType.LINEAR)\n",
+        "metric_y = {\"metric_id\": \"y\", \"goal\": \"MAXIMIZE\"}\n",
         "\n",
-        "study = Study.create_or_load(display_name=STUDY_DISPLAY_NAME, problem=problem)\n",
+        "study = {\n",
+        "    \"display_name\": STUDY_DISPLAY_NAME,\n",
+        "    \"study_spec\": {\n",
+        "        \"algorithm\": \"RANDOM_SEARCH\",\n",
+        "        \"parameters\": [param_x],\n",
+        "        \"metrics\": [metric_y],\n",
+        "    },\n",
+        "}\n",
         "\n",
+        "study = vizier_client.create_study(parent=PARENT, study=study)\n",
         "STUDY_NAME = study.name\n",
-        "print(\"STUDY_NAME: {}\".format(STUDY_NAME))"
+        "\n",
+        "print(STUDY_NAME)"
       ]
     },
     {
@@ -1567,7 +1586,9 @@
       "source": [
         "### Get Vizier study\n",
         "\n",
-        "You can get a study using the method `list()`."
+        "You can get a study using the method `get_study()`, with the following key/value pairs:\n",
+        "\n",
+        "- `name`: The name of the study."
       ]
     },
     {
@@ -1578,8 +1599,9 @@
       },
       "outputs": [],
       "source": [
-        "studies = Study.list()\n",
-        "print(studies[0].gca_resource)"
+        "study = vizier_client.get_study({\"name\": STUDY_NAME})\n",
+        "\n",
+        "print(study)"
       ]
     },
     {
@@ -1590,9 +1612,11 @@
       "source": [
         "### Get suggested trial\n",
         "\n",
-        "Next, query the Vizier service for a suggested trial(s) using the method `suggest()`, with the following key/value pairs:\n",
+        "Next, query the Vizier service for a suggested trial(s) using the method `suggest_trials`, with the following key/value pairs:\n",
         "\n",
-        "- `count`: The number of trials to suggest.\n",
+        "- `parent`: The name of the study.\n",
+        "- `suggestion_count`: The number of trials to suggest.\n",
+        "- `client_id`: blah\n",
         "\n",
         "This call is a long running operation. The method `result()` from the response object will wait until the call has completed."
       ]
@@ -1601,13 +1625,18 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "11ff2c4562cb"
+        "id": "vizier_suggest_trial"
       },
       "outputs": [],
       "source": [
         "SUGGEST_COUNT = 1\n",
+        "CLIENT_ID = \"1001\"\n",
         "\n",
-        "trials = study.suggest(count=SUGGEST_COUNT)\n",
+        "response = vizier_client.suggest_trials(\n",
+        "    {\"parent\": STUDY_NAME, \"suggestion_count\": SUGGEST_COUNT, \"client_id\": CLIENT_ID}\n",
+        ")\n",
+        "\n",
+        "trials = response.result().trials\n",
         "\n",
         "print(trials)\n",
         "\n",
@@ -1650,10 +1679,12 @@
       "source": [
         "RESULT = 0.01\n",
         "\n",
-        "measurement = pyvizier.Measurement()\n",
-        "measurement.metrics[\"y\"] = RESULT\n",
-        "\n",
-        "trials[0].add_measurement(measurement)"
+        "vizier_client.add_trial_measurement(\n",
+        "    {\n",
+        "        \"trial_name\": TRIAL_ID,\n",
+        "        \"measurement\": {\"metrics\": [{\"metric_id\": \"y\", \"value\": RESULT}]},\n",
+        "    }\n",
+        ")"
       ]
     },
     {
@@ -1664,7 +1695,7 @@
       "source": [
         "### Delete the Vizier study\n",
         "\n",
-        "The method 'delete()' will delete the study."
+        "The method 'delete_study()' will delete the study."
       ]
     },
     {
@@ -1675,7 +1706,7 @@
       },
       "outputs": [],
       "source": [
-        "study.delete()"
+        "vizier_client.delete_study({\"name\": STUDY_NAME})"
       ]
     },
     {

notebooks/community/ml_ops/stage2/get_started_with_logging.ipynb

@@ -73,7 +73,7 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to use Python and Cloud logging when training with `Vertex AI`.\n",
+        "In this tutorial, you learn how to use Python and Cloud logging awhen training with `Vertex AI`.\n",
         "\n",
         "This tutorial uses the following Google Cloud ML services:\n",
         "\n",

notebooks/community/ml_ops/stage2/mlops_experimentation.ipynb

@@ -39,15 +39,18 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/mlops_experimentation.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
+        "  <td>\n",
         "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/mlops_experimentation.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
-        "<br/><br/><br/>\n",
-        "\n",
-        "*Note: This notebook is not supported for execution in Colab*"
+        "<br/><br/><br/>"
       ]
     },
     {
@@ -213,18 +216,20 @@
         "\n",
         "ONCE_ONLY = False\n",
         "if ONCE_ONLY:\n",
-        "    ! pip3 install -U {USER_FLAG} -q tensorflow==2.5 \\\n",
-        "                                     tensorflow-data-validation==1.2 \\\n",
-        "                                     tensorflow-transform==1.2 \\\n",
-        "                                     tensorflow-io==0.18 \n",
-        "    \n",
-        "    ! pip3 install --upgrade {USER_FLAG} -q google-cloud-aiplatform[tensorboard] \\\n",
-        "                                            google-cloud-pipeline-components \\\n",
-        "                                            google-cloud-bigquery \\\n",
-        "                                            google-cloud-logging \\\n",
-        "                                            apache-beam[gcp] \\\n",
-        "                                            pyarrow \\\n",
-        "                                            cloudml-hypertune"
+        "    ! pip3 install -U tensorflow==2.5 $USER_FLAG -q\n",
+        "    ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG -q\n",
+        "    ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG -q\n",
+        "    ! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q\n",
+        "    ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q\n",
+        "    ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG -q\n",
+        "    ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
+        "    ! pip3 install --upgrade google-cloud-logging $USER_FLAG -q\n",
+        "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG -q\n",
+        "    ! pip3 install --upgrade pyarrow $USER_FLAG -q\n",
+        "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG -q\n",
+        "    ! pip3 install --upgrade kfp $USER_FLAG -q\n",
+        "    ! pip3 install --upgrade torchvision $USER_FLAG -q\n",
+        "    ! pip3 install --upgrade rpy2 $USER_FLAG -q"
       ]
     },
     {
@@ -439,11 +444,12 @@
         "import sys\n",
         "\n",
         "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "IS_COLAB = False\n",
         "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
         "    \"DL_ANACONDA_HOME\"\n",
         "):\n",
         "    if \"google.colab\" in sys.modules:\n",
+        "        IS_COLAB = True\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
         "        google_auth.authenticate_user()\n",

notebooks/community/ml_ops/stage3/README.md

@@ -33,144 +33,10 @@ The third stage in MLOps is formalization to develop an automated pipeline proce
 
 ### Get Started
 
+[Get started with AutoML Tabular Pipeline Workflows](get_started_with_automl_tabular_pipeline_workflow.ipynb)
 
-[Get started with Vertex AI Model Registry](community/ml_ops/stage3/get_started_with_model_registry.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Model Registry` to create and register multiple versions of a model.
-
+```
 The steps performed include:
-
-- Create and register a first version of a model to `Vertex AI Model Registry`.
-- Create and register a second version of a model to `Vertex AI Model Registry`.
-- Updating the model version which is the default (blessed).
-- Deleting a model version.
-- Retraining the next model version.
-
-[Get started with Dataflow pipeline components](community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb)
-
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataflow`.
-
-The steps performed include:
-
-- Build an Apache Beam data pipeline.
-- Encapsulate the Apache Beam data pipeline with a Dataflow component in a Vertex AI pipeline.
-- Execute a Vertex AI pipeline.
-
-[Get started with Apache Airflow and Vertex AI Pipelines](community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb)
-
-In this tutorial, you learn how to use Apache Airflow with `Vertex AI Pipelines`.
-
-The steps performed include:
-
-- Create Cloud Composer environment.
-- Upload Airflow DAG to Composer environment that performs data processing -- i.e., creates a BigQuery table from a CSV file.
-- Create a `Vertex AI Pipeline` that triggers the Airflow DAG.
-- Execute the `Vertex AI Pipeline`.
-
-[Get started with Kubeflow Pipelines](community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb)
-
-In this tutorial, you learn how to use `Kubeflow Pipelines`(KFP).
-
-The steps performed include:
-
-- Building KFP lightweight Python function components.
-- Assembling and compiling KFP components into a pipeline.
-- Executing a KFP pipeline using Vertex AI Pipelines.
-- Loading component and pipeline definitions from a source code repository.
-- Building sequential, parallel, multiple output components.
-- Building control flow into pipelines.
-
-[Get started with Vertex AI custom training pipeline components](community/ml_ops/stage3/get_started_with_custom_training_pipeline_components.ipynb)
-
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI Training`.
-
-The steps performed include:
-
-- Construct a pipeline for:
-    - Training a Vertex AI custom trained model.
-    - Test the serving binary with a batch prediction job.
-    - Deploying a Vertex AI custom trained model.
-- Execute a Vertex AI pipeline.
-- Construct a pipeline for:
-     - Construct a custom training component.
-     - Convert custom training component to CustomTrainingJobOp.
-     - Training a Vertex AI custom trained model using the converted component.
-    - Deploying a Vertex AI custom trained model.
-- Execute a Vertex AI pipeline.
-
-[Get started with Dataproc Serverless pipeline components](community/ml_ops/stage3/get_started_with_dataproc_serverless_pipeline_components.ipynb)
-
-
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataproc Serverless` service. 
-
-
-The steps performed include:
-
-- `DataprocPySparkBatchOp` for running PySpark batch workloads.
-- `DataprocSparkBatchOp` for running Spark batch workloads.
-- `DataprocSparkSqlBatchOp` for running Spark SQL batch workloads.
-- `DataprocSparkRBatchOp` for running SparkR batch workloads.
-
-[Get started with Vertex AI Hyperparameter Tuning pipeline components](community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb)
-
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI Hyperparameter Tuning`.
-
-The steps performed include:
-
-- Construct a pipeline for:
-    - Hyperparameter tune/train a custom model.
-    - Retrieve the tuned hyperparameter values and metrics to optimize.
-    - If the metrics exceed a specified threshold.
-      - Get the location of the model artifacts for the best tuned model.
-      - Upload the model artifacts to a `Vertex AI Model` resource.
-- Execute a Vertex AI pipeline.
-
-[Get started with machine management for Vertex AI Pipelines](community/ml_ops/stage3/get_started_with_machine_management.ipynb)
-
-In this tutorial, you convert a self-contained custom training component into a `Vertex AI CustomJob`, whereby:
-
-    - The training job and artifacts are trackable.
-    - Set machine resources, such as machine-type, cpu/gpu, memory, disk, etc.
-
-The steps performed in this tutorial include:
-
-- Create a custom component with a self-contained training job.
-- Execute pipeline using component-level settings for machine resources
-- Convert the self-contained training component into a `Vertex AI CustomJob`.
-- Execute pipeline using customjob-level settings for machine resources 
-
-[Get started with TFX pipelines](community/ml_ops/stage3/get_started_with_tfx_pipeline.ipynb)
-
-In this tutorial, you learn how to use TensorFlow Extended (TFX) with `Vertex AI Pipelines`.
-
-The steps performed include:
-
-- Create a TFX e2e pipeline.
-- Execute the pipeline locally.
-- Execute the pipeline on Google Cloud using `Vertex AI Training`
-- Execute the pipeline using `Vertex AI Pipelines`.
-
-[Get started with BigQuery ML pipeline components](community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb)
-
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `BigQuery ML`.
-
-The steps performed include:
-
-- Construct a pipeline for:
-    - Training BigQuery ML model.
-    - Evaluating the BigQuery ML model.
-    - Exporting the BigQuery ML model.
-    - Importing the BigQuery ML model to a Vertex AI model.
-    - Deploy the Vertex AI model.
-- Execute a Vertex AI pipeline.
-- Make a prediction with the deployed Vertex AI model.
-
-[Get started with AutoML tabular pipeline workflows](community/ml_ops/stage3/get_started_with_automl_tabular_pipeline_workflow.ipynb)
-
-In this tutorial, you learn how to use `AutoML Tabular Pipeline Template` for training, exporting and tuning an AutoML tabular model.
-
-The steps performed include:
-
 - Define training specification.
     - Dataset specification
     - Hyperparameter overide specification
@@ -183,41 +49,162 @@ The steps performed include:
 - Deploy exported OSS TF model.
 - Make a prediction.
 
-[Get started with rapid prototyping with AutoML and BigQuery ML](community/ml_ops/stage3/get_started_with_rapid_prototyping_bqml_automl.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Predictions` for rapid prototyping a model.
+[Get started with Vertex AI Model Registry](get_started_with_model_registry.ipynb)
 
+```
 The steps performed include:
+- Create and register a first version of a model to `Vertex AI Model Registry`.
+- Create and register a second version of a model to `Vertex AI Model Registry`.
+- Updating the model version which is the default (blessed).
+- Deleting a model version.
+- Retraining the next model version.
+```
 
-- Creating a BigQuery and Vertex AI training dataset.
-- Training a BigQuery ML and AutoML model.
-- Extracting evaluation metrics from the BigQueryML and AutoML models.
-- Selecting the best trained model.
-- Deploying the best trained model.
-- Testing the deployed model infrastructure.
-
-[Get started with AutoML pipeline components](community/ml_ops/stage3/get_started_with_automl_pipeline_components.ipynb)
-
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI AutoML`.
+[Get started with Dataproc serverless pipeline components](get_started_with_dataproc_serverless_pipeline_components.ipynb)
 
+```
 The steps performed include:
+- `DataprocPySparkBatchOp` for running PySpark batch workloads.
+- `DataprocSparkBatchOp` for running Spark batch workloads.
+- `DataprocSparkSqlBatchOp` for running Spark SQL batch workloads.
+- `DataprocSparkRBatchOp` for running SparkR batch workloads.
 
+```
+
+[Get started with TFX pipelines](get_started_with_tfx_pipeline.ipynb)
+
+```
+The steps performed include:
+- Create a TFX e2e pipeline.
+- Execute the pipeline locally.
+- Execute the pipeline on Google Cloud using `Vertex AI Training`
+- Execute the pipeline using `Vertex AI Pipelines`.
+```
+
+[Get started with Vertex AI Hyperparameter Tuning pipeline components](get_started_with_hpt_pipeline_components.ipynb)
+
+```
+The steps performed include:
+- Construct a pipeline for:
+    - Hyperparameter tune/train a custom model.
+    - Retrieve the tuned hyperparameter values and metrics to optimize.
+    - If the metrics exceed a specified threshold.
+      - Get the location of the model artifacts for the best tuned model.
+      - Upload the model artifacts to a `Vertex AI Model` resource.
+- Execute a Vertex AI pipeline.
+
+```
+
+[Get started with Apache Airflow and Vertex AI Pipelines](get_started_with_airflow_and_vertex_pipelines.ipynb)
+
+```
+The steps performed include:
+- Create Cloud Composer environment.
+- Upload Airflow DAG to Composer environment that performs data processing -- i.e., creates a BigQuery table from a CSV file.
+- Create a `Vertex AI Pipeline` that triggers the Airflow DAG.
+- Execute the `Vertex AI Pipeline`.
+
+```
+
+[Get started with Vertex AI custom training pipeline components](get_started_with_custom_training_pipeline_components.ipynb)
+
+```
+The steps performed include:
+- Construct a pipeline for:
+    - Training a Vertex AI custom trained model.
+    - Test the serving binary with a batch prediction job.
+    - Deploying a Vertex AI custom trained model.
+- Execute a Vertex AI pipeline.
+- Construct a pipeline for:
+     - Construct a custom training component.
+     - Convert custom training component to CustomTrainingJobOp.
+     - Training a Vertex AI custom trained model using the converted component.
+    - Deploying a Vertex AI custom trained model.
+- Execute a Vertex AI pipeline.
+```
+
+[Get started with AutoML pipeline components](get_started_with_automl_pipeline_components.ipynb)
+
+```
+The steps performed include:
 - Construct a pipeline for:
     - Training a Vertex AI AutoML trained model.
     - Test the serving binary with a batch prediction job.
     - Deploying a Vertex AI AutoML trained model.
 - Execute a Vertex AI pipeline.
 
+```
 
-[Get started with BigQuery and TFDV pipeline components](community/ml_ops/stage3/get_started_with_bq_tfdv_pipeline_components.ipynb)
-
-In this tutorial, you learn how to use build lightweight Python components for BigQuery and TensorFlow Data Validation.
+[Get started with Kubeflow pipelines](get_started_with_kubeflow_pipelines.ipynb)
 
+```
 The steps performed include:
+- Building KFP lightweight Python function components.
+- Assembling and compiling KFP components into a pipeline.
+- Executing a KFP pipeline using Vertex AI Pipelines.
+- Loading component and pipeline definitions from a source code repository.
+- Building sequential, parallel, multiple output components.
+- Building control flow into pipelines.
 
+```
+
+[Get started with machine management for Vertex AI Pipelines](get_started_with_machine_management.ipynb)
+
+```
+The steps performed in this tutorial include:
+- Create a custom component with a self-contained training job.
+- Execute pipeline using component-level settings for machine resources
+- Convert the self-contained training component into a `Vertex AI CustomJob`.
+- Execute pipeline using customjob-level settings for machine resources 
+
+```
+
+[Get started with BigQuery and TFDV pipeline components](get_started_with_bq_tfdv_pipeline_components.ipynb)
+
+```
+The steps performed include:
 - Build and execute a pipeline component for creating a Vertex AI Tabular Dataset from a BigQuery table.
 - Build and execute a pipeline component for generating TFDV statistics and schema from a Vertex AI Tabular Dataset.
 - Execute a Vertex AI pipeline.
+```
+
+[Get started with Dataflow pipeline components](get_started_with_dataflow_pipeline_components.ipynb)
+
+```
+The steps performed include:
+- Build an Apache Beam data pipeline.
+- Encapsulate the Apache Beam data pipeline with a Dataflow component in a Vertex AI pipeline.
+- Execute a Vertex AI pipeline.
+```
+
+[Get started with BigQuery ML pipeline components](get_started_with_bqml_pipeline_components.ipynb)
+
+```
+The steps performed include:
+- Construct a pipeline for:
+    - Training BigQuery ML model.
+    - Evaluating the BigQuery ML model.
+    - Exporting the BigQuery ML model.
+    - Importing the BigQuery ML model to a Vertex AI model.
+    - Deploy the Vertex AI model.
+- Execute a Vertex AI pipeline.
+- Make a prediction with the deployed Vertex AI model.
+```
+
+[Get started with rapid prototyping with AutoML and BigQuery ML](get_started_with_rapid_prototyping_bqml_automl.ipynb)
+
+```
+The steps performed include:
+- Creating a BigQuery and Vertex AI training dataset.
+- Training a BigQuery ML and AutoML model.
+- Extracting evaluation metrics from the BigQueryML and AutoML models.
+- Selecting the best trained model.
+- Deploying the best trained model.
+- Testing the deployed model infrastructure.
+```
+
 
 ### E2E Stage Example
 

notebooks/community/ml_ops/stage3/get_started_with_machine_management.ipynb

@@ -81,7 +81,7 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to convert a self-contained custom training component into a `Vertex AI CustomJob`, whereby:\n",
+        "In this tutorial, you convert a self-contained custom training component into a `Vertex AI CustomJob`, whereby:\n",
         "\n",
         "    - The training job and artifacts are trackable.\n",
         "    - Set machine resources, such as machine-type, cpu/gpu, memory, disk, etc.\n",
@@ -569,6 +569,7 @@
       "source": [
         "import json\n",
         "\n",
+        "import numpy as np\n",
         "from google.cloud import aiplatform\n",
         "from google_cloud_pipeline_components.v1.custom_job import \\\n",
         "    create_custom_training_job_from_component\n",
@@ -796,6 +797,7 @@
         "    epochs: int,\n",
         ") -> str:\n",
         "    import numpy as np\n",
+        "    import tensorflow as tf\n",
         "\n",
         "    def get_data():\n",
         "        from tensorflow.keras.datasets import mnist\n",
@@ -902,7 +904,7 @@
         "        },\n",
         "    ).after(training_job_task)\n",
         "\n",
-        "    _ = ModelUploadOp(\n",
+        "    model_upload = ModelUploadOp(\n",
         "        project=project,\n",
         "        display_name=\"mnist_model\",\n",
         "        unmanaged_container_model=import_unmanaged_model_task.outputs[\"artifact\"],\n",
@@ -1178,7 +1180,7 @@
         "        },\n",
         "    ).after(training_job_task)\n",
         "\n",
-        "    _ = ModelUploadOp(\n",
+        "    model_upload = ModelUploadOp(\n",
         "        project=project,\n",
         "        display_name=\"mnist_model\",\n",
         "        unmanaged_container_model=import_unmanaged_model_task.outputs[\"artifact\"],\n",

notebooks/community/ml_ops/stage4/README.md

@@ -42,192 +42,104 @@ This stage may be done entirely by MLOps. We recommend:
 
 ### Get Started
 
+[Get started with Vertex Explainable AI](get_started_with_vertex_xai.ipynb)
 
-[Get started with Vertex AI Model Registry](community/ml_ops/stage3/get_started_with_model_registry.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Model Registry` to create and register multiple versions of a model.
-
+```
 The steps performed include:
+- Train an AutoML tabular model.
+    - Do a batch prediction with explanations.
+    - Do an online prediction with explanations.
+- Train an custom TensorFlow tabular model.
+    - Manually set configuration metadata.
+    - Do a batch prediction with explanations.
+    - Do an online prediction with explanations.
+    - Automatically set configuration metadata.
+- Train an custom TensorFlow image model.
+    - Manually set configuration metadata.
+    - Do a batch prediction with explanations.
+    - Do an online prediction with explanations.
+- Train an custom XGBoost tabular model.
+    - Manually set configuration metadata.
+    - Do an online prediction with explanations.
+- Train an custom scikit-learn tabular model.
+    - Manually set configuration metadata.
+    - Do an online prediction with explanations.
 
-- Create and register a first version of a model to `Vertex AI Model Registry`.
-- Create and register a second version of a model to `Vertex AI Model Registry`.
-- Updating the model version which is the default (blessed).
-- Deleting a model version.
-- Retraining the next model version.
+```
 
-[Get started with Dataflow pipeline components](community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb)
-
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataflow`.
+[Get started with Google Artifact Registry](get_started_with_google_artifact_registry.ipynb)
 
+```
 The steps performed include:
+- Creating a private Docker repository.
+- Tagging a container image, specific to the private Docker repository.
+- Pushing a container image to the private Docker repository.
+- Pulling a container image from the private Docker repository.
+- Deleting a private Docker repository.
+```
 
-- Build an Apache Beam data pipeline.
-- Encapsulate the Apache Beam data pipeline with a Dataflow component in a Vertex AI pipeline.
-- Execute a Vertex AI pipeline.
-
-[Get started with Apache Airflow and Vertex AI Pipelines](community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb)
-
-In this tutorial, you learn how to use Apache Airflow with `Vertex AI Pipelines`.
+[Get started with AutoML training and ML Metadata](get_started_with_vertex_ml_metadata_and_automl.ipynb)
 
+```
 The steps performed include:
+- Create a `Dataset` resource.
+- Create a corresponding `google.VertexDataset` artifact.
+- Train a model using `AutoML`.
+- Create a corresponding `google.VertexModel` artifact.
+- Create an `Endpoint` resource.
+- Create a corresponding `google.Endpoint` artifact.
+- Deploy the train model to the `Endpoint`.
+- Create an execution and context for the `AutoML` training job and deployment.
+- Add the corresponding artifacts and context to the execution.
+- Add artifact links (event) to the execution.
+- Display the execution graph.
+```
 
-- Create Cloud Composer environment.
-- Upload Airflow DAG to Composer environment that performs data processing -- i.e., creates a BigQuery table from a CSV file.
-- Create a `Vertex AI Pipeline` that triggers the Airflow DAG.
-- Execute the `Vertex AI Pipeline`.
-
-[Get started with Kubeflow Pipelines](community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb)
-
-In this tutorial, you learn how to use `Kubeflow Pipelines`(KFP).
+[Get started with Vertex AI ML Metadata](get_started_with_vertex_ml_metadata.ipynb)
 
+```
 The steps performed include:
+- Create a `Metadatastore` resource.
+- Create (record)/List an `Artifact`, with artifacts and metadata.
+- Create (record)/List an `Execution`.
+- Create (record)/List a `Context`.
+- Add `Artifact` to `Execution` as events.
+- Add `Execution` and `Artifact` into the `Context`
+- Delete `Artifact`, `Execution` and `Context`.
+- Create and run a `Vertex AI Pipeline` ML workflow to train and deploy a scikit-learn model.
+    - Create custom pipeline components that generate artifacts and metadata.
+    - Compare Vertex AI Pipelines runs.
+    - Trace the lineage for pipeline-generated artifacts.
+    - Query your pipeline run metadata.
+```
 
-- Building KFP lightweight Python function components.
-- Assembling and compiling KFP components into a pipeline.
-- Executing a KFP pipeline using Vertex AI Pipelines.
-- Loading component and pipeline definitions from a source code repository.
-- Building sequential, parallel, multiple output components.
-- Building control flow into pipelines.
-
-[Get started with Vertex AI custom training pipeline components](community/ml_ops/stage3/get_started_with_custom_training_pipeline_components.ipynb)
-
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI Training`.
+[Get started with Vertex AI Model Evaluation](get_started_with_model_evaluation.ipynb)
 
+```
 The steps performed include:
-
-- Construct a pipeline for:
-    - Training a Vertex AI custom trained model.
-    - Test the serving binary with a batch prediction job.
-    - Deploying a Vertex AI custom trained model.
-- Execute a Vertex AI pipeline.
-- Construct a pipeline for:
-     - Construct a custom training component.
-     - Convert custom training component to CustomTrainingJobOp.
-     - Training a Vertex AI custom trained model using the converted component.
-    - Deploying a Vertex AI custom trained model.
-- Execute a Vertex AI pipeline.
-
-[Get started with Dataproc Serverless pipeline components](community/ml_ops/stage3/get_started_with_dataproc_serverless_pipeline_components.ipynb)
-
-
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataproc Serverless` service. 
-
-
-The steps performed include:
-
-- `DataprocPySparkBatchOp` for running PySpark batch workloads.
-- `DataprocSparkBatchOp` for running Spark batch workloads.
-- `DataprocSparkSqlBatchOp` for running Spark SQL batch workloads.
-- `DataprocSparkRBatchOp` for running SparkR batch workloads.
-
-[Get started with Vertex AI Hyperparameter Tuning pipeline components](community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb)
-
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI Hyperparameter Tuning`.
-
-The steps performed include:
-
-- Construct a pipeline for:
-    - Hyperparameter tune/train a custom model.
-    - Retrieve the tuned hyperparameter values and metrics to optimize.
-    - If the metrics exceed a specified threshold.
-      - Get the location of the model artifacts for the best tuned model.
-      - Upload the model artifacts to a `Vertex AI Model` resource.
-- Execute a Vertex AI pipeline.
-
-[Get started with machine management for Vertex AI Pipelines](community/ml_ops/stage3/get_started_with_machine_management.ipynb)
-
-In this tutorial, you convert a self-contained custom training component into a `Vertex AI CustomJob`, whereby:
-
-    - The training job and artifacts are trackable.
-    - Set machine resources, such as machine-type, cpu/gpu, memory, disk, etc.
-
-The steps performed in this tutorial include:
-
-- Create a custom component with a self-contained training job.
-- Execute pipeline using component-level settings for machine resources
-- Convert the self-contained training component into a `Vertex AI CustomJob`.
-- Execute pipeline using customjob-level settings for machine resources 
-
-[Get started with TFX pipelines](community/ml_ops/stage3/get_started_with_tfx_pipeline.ipynb)
-
-In this tutorial, you learn how to use TensorFlow Extended (TFX) with `Vertex AI Pipelines`.
-
-The steps performed include:
-
-- Create a TFX e2e pipeline.
-- Execute the pipeline locally.
-- Execute the pipeline on Google Cloud using `Vertex AI Training`
-- Execute the pipeline using `Vertex AI Pipelines`.
-
-[Get started with BigQuery ML pipeline components](community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb)
-
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `BigQuery ML`.
-
-The steps performed include:
-
-- Construct a pipeline for:
-    - Training BigQuery ML model.
-    - Evaluating the BigQuery ML model.
-    - Exporting the BigQuery ML model.
-    - Importing the BigQuery ML model to a Vertex AI model.
-    - Deploy the Vertex AI model.
-- Execute a Vertex AI pipeline.
-- Make a prediction with the deployed Vertex AI model.
-
-[Get started with AutoML tabular pipeline workflows](community/ml_ops/stage3/get_started_with_automl_tabular_pipeline_workflow.ipynb)
-
-In this tutorial, you learn how to use `AutoML Tabular Pipeline Template` for training, exporting and tuning an AutoML tabular model.
-
-The steps performed include:
-
-- Define training specification.
-    - Dataset specification
-    - Hyperparameter overide specification
-    - machine specifications
-- Construct tabular workflow pipeline.
-- Compile and execute pipeline.
-- View evaluation metrics artifact.
-- Export AutoML model as an OSS TF model.
-- Create `Endpoint` resource.
-- Deploy exported OSS TF model.
-- Make a prediction.
-
-[Get started with rapid prototyping with AutoML and BigQuery ML](community/ml_ops/stage3/get_started_with_rapid_prototyping_bqml_automl.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Predictions` for rapid prototyping a model.
-
-The steps performed include:
-
-- Creating a BigQuery and Vertex AI training dataset.
-- Training a BigQuery ML and AutoML model.
-- Extracting evaluation metrics from the BigQueryML and AutoML models.
-- Selecting the best trained model.
-- Deploying the best trained model.
-- Testing the deployed model infrastructure.
-
-[Get started with AutoML pipeline components](community/ml_ops/stage3/get_started_with_automl_pipeline_components.ipynb)
-
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI AutoML`.
-
-The steps performed include:
-
-- Construct a pipeline for:
-    - Training a Vertex AI AutoML trained model.
-    - Test the serving binary with a batch prediction job.
-    - Deploying a Vertex AI AutoML trained model.
-- Execute a Vertex AI pipeline.
-
-
-[Get started with BigQuery and TFDV pipeline components](community/ml_ops/stage3/get_started_with_bq_tfdv_pipeline_components.ipynb)
-
-In this tutorial, you learn how to use build lightweight Python components for BigQuery and TensorFlow Data Validation.
-
-The steps performed include:
-
-- Build and execute a pipeline component for creating a Vertex AI Tabular Dataset from a BigQuery table.
-- Build and execute a pipeline component for generating TFDV statistics and schema from a Vertex AI Tabular Dataset.
-- Execute a Vertex AI pipeline.
-
+- Evaluate an `AutoML` model.
+    - Train an `AutoML` image classification model.
+    - Retrieve the default evaluation metrics from training.
+    - Do a batch evaluation for a custom evaluation slice.
+- Evaluate a BigQuery ML model.
+    - Train a `BigQuery ML` tabular classification model.
+    - Retrieve the default evaluation metrics from training.
+    - Do a batch evaluation for a custom evaluation slice.
+- Evaluate a custom model.
+    - Do a batch evaluation for a custom evaluation slice.
+    - Add an evaluation to the `Model Registry` for the `Model` resource.
+- Evaluate an `AutoML` model.
+    - Train an `AutoML` image classification model.
+    - Retrieve the default evaluation metrics from training.
+    - Do a batch evaluation for a custom evaluation slice.
+- Evaluate a BigQuery ML model.
+    - Train a `BigQuery ML` tabular classification model.
+    - Retrieve the default evaluation metrics from training.
+    - Do a batch evaluation for a custom evaluation slice.
+- Evaluate a custom model.
+    - Do a batch evaluation for a custom evaluation slice.
+    - Add an evaluation to the `Model Registry` for the `Model` resource.
+```
 ### E2E Stage Example
 
 Stage 4: Evaluation

notebooks/community/ml_ops/stage5/README.md

@@ -25,12 +25,10 @@ The fifth stage in MLOps is deployment to production of the blessed model, which
 ### Get Started
 
 
-[Get started with Vertex AI Endpoints](community/ml_ops/stage5/get_started_with_vertex_endpoints.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Endpoint` resources.
+[Get started with Vertex AI Endpoints](get_started_with_vertex_endpoints.ipynb)
 
+```
 The steps performed include:
-
 - Creating an `Endpoint` resource.
 - List all `Endpoint` resources.
 - List `Endpoint` resources by query filter.
@@ -45,29 +43,12 @@ The steps performed include:
 - Delete an `Endpoint` resource.
 - In pipeline: Create an `Endpoint` resource and deploy an existing `Model` resource to the `Endpoint` resource.
 - In pipeline: Deploy an existing `Model` resource to an existing `Endpoint` resource.
+```
 
-[Get started with Vertex AI Endpoint and shared VM](community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb)
-
-In this tutorial, you learn how to use deployment resource pools for deploying models. A deployment resouce pool provides one with the ability to co-host more than one model on the same (shared) VM.
+[Get started with configuring autoscaling for Vertex AI Endpoint deployment](get_started_with_autoscaling.ipynb)
 
+```
 The steps performed include:
-
-- Upload a pre-trained image classification model as a `Model` resource (model A).
-- Upload a pre-trained text sentence encoder model as a `Model` resource (model B).
-- Create a shared VM deployment resource pool.
-- List shared VM deployment resource pools.
-- Create two `Endpoint` resources.
-- Deploy first model (model A) to first `Endpoint` resource using deployment resource pool.
-- Deploy second model (model B) to second `Endpoint` resource using deployment resource pool.
-- Make a prediction request with first deployed model (model A).
-- Make a prediction request with second deployed model (model B).
-
-[Get started with configuring autoscaling for Vertex AI Endpoint deployment](community/ml_ops/stage5/get_started_with_autoscaling.ipynb)
-
-In this tutorial, you learn how to use fine-tune control auto-scaling configuration when deploying a `Model` resource to an `Endpoint` resource.
-
-The steps performed include:
-
 - Download a pretrained image classification model from TensorFlow Hub.
 - Upload the pretrained model as a `Model` resource.
 - Create an `Endpoint` resource.
@@ -77,20 +58,31 @@ The steps performed include:
 - Fine-tune scaling thresholds for CPU utilization.
 - Fine-tune scaling thresholds for GPU utilization.
 - Deploy mix of CPU and GPU model instances with auto-scaling to an `Endpoint` resource.
+```
 
-[Get started with Vertex AI Private Endpoints](community/ml_ops/stage5/get_started_with_vertex_private_endpoints.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Private Endpoint` resources.
+[Get started with Vertex AI Private Endpoints](get_started_with_vertex_private_endpoints.ipynb)
 
+```
 The steps performed include:
-
 - Creating a `Private Endpoint` resource.
 - Configure a VPC peering connection.
 - Configuring the serving binary of a `Model` resource for deployment to a `Private Endpoint` resource.
 - Deploying a `Model` resource to a `Private Endpoint` resource.
 - Send a prediction request to a `Private Endpoint`
-- Enable two additional APIs: Service Networking and Cloud DNS.
-- Add Compute Admin Network role to your (default) service account.
-- Issue two gcloud commands to setup the VPC peering for your service account.
-- There is *currently* no SDK support yet, so private endpoint is created with GAPIC client and has an extra argument for the peering network.
-- To send a request, you can't use SDK/GAPIC since they do a HTTP internet request. Instead, you use curl to send a peer-to-peer request.
+
+```
+
+[Get started with Vertex AI Endpoint and shared VM](get_started_with_vertex_endpoint_and_shared_vm.ipynb)
+
+```
+The steps performed include:
+- Upload a pre-trained image classification model as a `Model` resource (model A).
+- Upload a pre-trained text sentence encoder model as a `Model` resource (model B).
+- Create a shared VM deployment resource pool.
+- List shared VM deployment resource pools.
+- Create two `Endpoint` resources.
+- Deploy first model (model A) to first `Endpoint` resource using deployment resource pool.
+- Deploy second model (model B) to second `Endpoint` resource using deployment resource pool.
+- Make a prediction request with first deployed model (model A).
+- Make a prediction request with second deployed model (model B).
+```

notebooks/community/ml_ops/stage6/README.md

@@ -30,232 +30,48 @@ This stage may be done entirely by MLOps. We recommend:
 ### Get Started
 
 
-[Get started with Vertex AI Batch Prediction for AutoML image models](community/ml_ops/stage6/get_started_with_automl_image_model_batch.ipynb)
-
-In this tutorial, you create an AutoML image classification model from a Python script, and then do a batch prediction using the Vertex AI SDK. 
+[Get started with TensorFlow serving functions with Vertex AI Prediction](get_started_with_tf_serving_function.ipynb)
 
+```
 The steps performed include:
-
-- Create a Vertex `Dataset` resource.
-- Train an `AutoML` image classification model.
-- Make a batch prediction with JSONL input.
-
-[Get started with Vertex AI Matching Engine and Swivel builtin algorithm](community/ml_ops/stage6/get_started_with_matching_engine_swivel.ipynb)
-
-In this notebook, you learn how to train custom embeddings using Vertex AI Pipelines and subsequently train and deploy a matching engine index using the embeddings.
-
-The steps performed include:
-
-1. Train the `Swivel` algorithm to generate embeddings (encoder) for the dataset.
-2. Make example predictions (embeddings) from then trained encoder.
-3. Generate embeddings using the trained `Swivel` builtin algorithm.
-4. Store embeddings to format supported by `Matching Engine`.
-5. Create a `Matching Engine Index` for the embeddings.
-6. Deploy the `Matching Engine Index` to a `Index Endpoint`.
-7. Make a matching engine prediction request.
-
-[Get started with Vertex AI Matching Engine](community/ml_ops/stage6/get_started_with_matching_engine.ipynb)
-
-In this notebook, you learn how to create Approximate Nearest Neighbor (ANN) Index, query against indexes. 
-
-The steps performed include:
-
-- Create ANN Index.
-- Create an IndexEndpoint with VPC Network
-- Deploy ANN Index
-- Perform online query
-- Deploy brute force Index.
-- Perform calibration between ANN and brute force index.
-
-[Get started with Vertex AI Matching Engine and Two Towers builtin algorithm](community/ml_ops/stage6/get_started_with_matching_engine_twotowers.ipynb)
-
-
-In this notebook, you learn how to use the `Two-Tower` builtin algorithms for generating embeddings for a dataset, for use with generating an `Matching Engine Index`, with the `Vertex AI Matching Engine` service.
-
-The steps performed include:
-
-1. Train the `Two-Tower` algorithm to generate embeddings (encoder) for the dataset.
-2. Hyperparameter tune the trained `Two-Tower` encoder.
-3. Make example predictions (embeddings) from then trained encoder.
-4. Generate embeddings using the trained `Two-Tower` builtin algorithm.
-5. Store embeddings to format supported by `Matching Engine`.
-6. Create a `Matching Engine Index` for the embeddings.
-7. Deploy the `Matching Engine Index` to a `Index Endpoint`.
-8. Make a matching engine prediction request.
-
-[Get started with Vertex AI Batch Prediction for custom tabular models](community/ml_ops/stage6/get_started_with_custom_tabular_model_batch.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Batch Prediction`  with a custom tabular model.
-
-The steps performed include:
-
-- Upload a pretrained tabular model as a `Vertex AI Model` resource.
-- Make batch prediction to the `Model` resource, in JSONL format.
-- Make batch prediction to the `Model` resource, in CSV format.
-- Make batch prediction to the `Model` resource, in BigQuery format.
-
-[Get started with Optimized TensorFlow Enterprise container with Vertex AI Prediction / text models](community/ml_ops/stage6/get_started_with_optimized_tfe_bert.ipynb)
-
-In this tutorial, you learn how to use `TensorFlow Enterprise Optimized` container for TensorFlow models deployed to a `Vertex AI Endpoint` resource.
-
-The steps performed include:
-
-- Download a pretrained BERT model from TensorFlow Hub.
-- Fine-tune (transfer learning) the BERT model as a binary classifier.
-- Upload the TensorFlow Hub model as a `Vertex AI Model` resource, with standard TensorFlow serving container.
-- Upload the TensorFlow Hub model as a `Vertex AI Model` resource, with TensorFlow Enterprise Optimized container
-- Create two `Endpoint` resources.
-- Deploying both `Model` resources to separate `Endpoint` resources.
-- Make the same online prediction requests to both `Model` resource instances deployed to the `Endpoint` resources.
-- Compare the prediction accuracy between the two deployed `Model` resources.
-- Configuring container settings for fine-tune control of optimizations.
-- Create a `Private Endpoint` resource.
-- Deploy the `Model` resoure with then `TensorFlow Enterprise Optimized` to the `Private Endpoint` resource.
-- Make an online prediction request to the `Private Endpoint` resource.
-
-[Get started with Vertex AI Batch Prediction and Explainable AI for AutoML tabular models](community/ml_ops/stage6/get_started_with_automl_tabular_model_batch.ipynb)
-
-In this tutorial, you create an AutoML tabular binary classification model from a Python script, and then do a batch prediction with Explainable AI using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.
-
-The steps performed include:
-
-- Create a Vertex `Dataset` resource.
-- Train an `AutoML` tabular model.
-- Make a batch prediction with CSV input.
-- Make a batch prediction with JSONL objects input.
-- Make a batch prediction with JSONL list input.
-- Make a batch prediction with BigQuery table input.
-- Make a batch prediction with explanations.
-
-[Get started with re-importing AutoML tabular models](community/ml_ops/stage6/get_started_with_automl_tabular_exported_deploy.ipynb)
-
-In this tutorial, you learn how to use `AutoML Tabular` for re-importing exported model artifacts as a `Model` resource. This is useful for example, if one wants to move the exported model across projects.
-
-The steps performed include:
-
-- Importing a pretrained AutoML tabular exported model artifacts, as a `Model` resource.
-- Create an `Endpoint` resource.
-- Deploy the `Model` resource to the `Endpoint` resource.
-- Make a prediction.
-
-[Get started with Vertex AI Batch Prediction for AutoML text models](community/ml_ops/stage6/get_started_with_automl_text_model_batch.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Batch Prediction`  with a `AutoML` text model.
-
-The steps performed include:
-
-- Create a Vertex `Dataset` resource.
-- Train an `AutoML` model.
-- Make a batch prediction with JSONL input
-
-[Get started with Vertex AI Prediction for AutoML text models](community/ml_ops/stage6/get_started_with_automl_text_model_online.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Prediction`  with a `AutoML` text model.
-
-The steps performed include:
-
-- Create a Vertex `Dataset` resource.
-- Train an `AutoML` model.
-- Deploy the model to an `Endpoint` resource.
-- Make an online prediction.
-
-[Get started with TensorFlow serving functions with Vertex AI Raw Prediction](community/ml_ops/stage6/get_started_with_raw_predict.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Raw Prediction` on a `Vertex AI Endpoint` resource.
-
-The steps performed include:
-
-- Download a pretrained tabular classification model artifacts for a TensorFlow 1.x estimator.
-- Upload the TensorFlow estimator model as a `Vertex AI Model` resource.
-- Creating an `Endpoint` resource.
-- Deploying the `Model` resource to an `Endpoint` resource.
-- Make an online raw prediction to the `Model` resource instance deployed to the `Endpoint` resource.
-
-[Get started with TensorFlow serving functions with Vertex AI Prediction](community/ml_ops/stage6/get_started_with_tf_serving_function.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Prediction` on a `Vertex AI Endpoint` resource with a serving function.
-
-The steps performed include:
-
 - Download a pretrained image classification model from TensorFlow Hub.
 - Create a serving function to receive compressed image data, and output decomopressed preprocessed data for the model input.
 - Upload the TensorFlow Hub model and serving function as a `Vertex AI Model` resource.
 - Creating an `Endpoint` resource.
 - Deploying the `Model` resource to an `Endpoint` resource.
 - Make an online prediction to the `Model` resource instance deployed to the `Endpoint` resource.
+```
 
-[Get started with Vertex Explainable AI using custom deployment container](community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb)
-
-In this tutorial, you learn to build a custom container to serve a PyTorch model on `Vertex AI Endpoint`. 
+[Get started with FastAPI with Vertex AI Prediction](get_started_with_fastapi.ipynb)
 
+```
 The steps performed include:
-
-- Locally train a Pytorch tabular classifier.
-- Locally test the trained model.
-- Build a HTTP server using FastAPI.
-- Create a custom serving container with the trained model and FastAPI server.
-- Locally test the custom serving container.
-- Push the custom serving container to the Artifact Registry.
-- Upload the custom serving container as a `Model` resource.
-- Deploy the `Model` resource to an `Endpoint` resource.
-- Make a prediction request to the deployed custom serving container.
-- Make an explanation request to the deployed custom serving container.
-
-[Get started with Vertex AI Online Prediction for AutoML image models](community/ml_ops/stage6/get_started_with_automl_image_model_online.ipynb)
-
-In this tutorial, you create an AutoML image classification model from a Python script, and then do an online prediction using the Vertex AI SDK. 
-
-The steps performed include:
-
-- Create a Vertex `Dataset` resource.
-- Train an `AutoML` image classification model.
-- Make an online prediction.
-
-[Get started with FastAPI with Vertex AI Prediction](community/ml_ops/stage6/get_started_with_fastapi.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Prediction` on a `Vertex AI Endpoint` with a custom serving binary using `FastAPI`.
-
-The steps performed include:
-
 - Download a pretrained image classification model from TensorFlow Hub.
 - Create a serving function to receive compressed image data, and output decomopressed preprocessed data for the model input.
 - Upload the TensorFlow Hub model and serving function as a `Vertex AI Model` resource.
 - Creating an `Endpoint` resource.
 - Deploying the `Model` resource to an `Endpoint` resource with `FastAPI` custom serving binary.
 - Make an online prediction to the `Model` resource instance deployed to the `Endpoint` resource.
+```
 
-[Get started with Vertex AI Online Prediction for AutoML tabular models](community/ml_ops/stage6/get_started_with_automl_tabular_model_online.ipynb)
+[Get started with Nvidia Triton server](get_started_with_nvidia_triton_serving.ipynb)
 
-In this tutorial, you create an AutoML tabular binary classification model from a Python script, and then do an online prediction using the Vertex AI SDK. 
+```
+The steps performed in this tutorial include:
+- Download the model artifacts from TensorFlow Hub.
+- Create Triton serving configuration file for the model.
+- Construct a custom container, with Triton serving image, for model deployment.
+- Upload the model as a `Vertex AI Model` resource.
+- Deploy the `Vertex AI Model` resource to a `Vertex AI Endpoint` resource.
+- Make a prediction request
+- Undeploy the `Model` resource and delete the `Endpoint`
 
+```
+
+[Get started with Custom Prediction Routine (CPR)](get_started_with_cpr.ipynb)
+
+```
 The steps performed include:
-
-- Create a Vertex `Dataset` resource.
-- Train an `AutoML` tabular model.
-- Deploy the model to an `Endpoint` resource.
-- Make an online prediction.
-- Make an online prediction with explanations.
-
-[Get started with TensorFlow Serving with Vertex AI Prediction](community/ml_ops/stage6/get_started_with_tf_serving.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Prediction` on a `Vertex AI Endpoint` resource with `TensorFlow Serving` serving binary.
-
-The steps performed include:
-
-- Download a pretrained image classification model from TensorFlow Hub.
-- Create a serving function to receive compressed image data, and output decomopressed preprocessed data for the model input.
-- Upload the TensorFlow Hub model and serving function as a `Vertex AI Model` resource.
-- Creating an `Endpoint` resource.
-- Deploying the `Model` resource to an `Endpoint` resource with `TensorFlow Serving` serving binary.
-- Make an online prediction to the `Model` resource instance deployed to the `Endpoint` resource.
-- Make a batch prediction to the `Model` resource instance.
-
-[Get started with Custom Prediction Routine (CPR)](community/ml_ops/stage6/get_started_with_cpr.ipynb)
-
-In this tutorial, you learn how to use Custom Prediction Routine (CPR) for `Vertex AI Predictions`.
-
-The steps performed include:
-
 - Write a custom data preprocessor.
 - Train the model.
 - Build a custom scikit-learn serving container with custom data preprocessing using the Custom Prediction Routine model server.
@@ -277,50 +93,115 @@ The steps performed include:
     - Test the model serving container locally.
     - Upload and deploy the model serving container to Vertex AI Endpoint.
     - Make a prediction request.
+```
 
-[Get started with Vertex AI Batch Prediction for custom text models](community/ml_ops/stage6/get_started_with_custom_text_model_batch.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Batch Prediction`  with a custom text model.
+[Get started with re-importing AutoML tabular models](get_started_automl_tabular_exported_deploy.ipynb)
 
+```
 The steps performed include:
+- Importing a pretrained AutoML tabular exported model artifacts, as a `Model` resource.
+- Create an `Endpoint` resource.
+- Deploy the `Model` resource to the `Endpoint` resource.
+- Make a prediction.
+```
 
-- Download a pretrained TensorFlow RNN model.
-- Upload the pretrained model as a `Vertex AI Model` resource.
-- Make batch prediction to the `Model` resource, in JSONL format.
-
-[Get started with NVIDIA Triton server](community/ml_ops/stage6/get_started_with_nvidia_triton_serving.ipynb)
-
-In this tutorial, you deploy a container running Nvidia Triton Server with a `Vertex AI Model` resource to a `Vertex AI Endpoint` for making online predictions.
-
-The steps performed in this tutorial include:
-
-- Download the model artifacts from TensorFlow Hub.
-- Create Triton serving configuration file for the model.
-- Construct a custom container, with Triton serving image, for model deployment.
-- Upload the model as a `Vertex AI Model` resource.
-- Deploy the `Vertex AI Model` resource to a `Vertex AI Endpoint` resource.
-- Make a prediction request
-- Undeploy the `Model` resource and delete the `Endpoint`
-
-[Get started with Vertex AI Batch Prediction for custom image models](community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Batch Prediction`  with a custom image model.
+[Get started with Vertex Explainable AI using custom deployment container](get_started_with_xai_and_custom_server.ipynb)
 
+```
 The steps performed include:
+- Locally train a Pytorch tabular classifier.
+- Locally test the trained model.
+- Build a HTTP server using FastAPI.
+- Create a custom serving container with the trained model and FastAPI server.
+- Locally test the custom serving container.
+- Push the custom serving container to the Artifact Registry.
+- Upload the custom serving container as a `Model` resource.
+- Deploy the `Model` resource to an `Endpoint` resource.
+- Make a prediction request to the deployed custom serving container.
+- Make an explanation request to the deployed custom serving container.
 
+```
+
+[Get started with TensorFlow serving functions with Vertex AI Raw Prediction](get_started_with_raw_predict.ipynb)
+
+```
+The steps performed include:
+- Download a pretrained tabular classification model artifacts for a TensorFlow 1.x estimator.
+- Upload the TensorFlow estimator model as a `Vertex AI Model` resource.
+- Creating an `Endpoint` resource.
+- Deploying the `Model` resource to an `Endpoint` resource.
+- Make an online raw prediction to the `Model` resource instance deployed to the `Endpoint` resource.
+```
+
+[Get started with Vertex AI Matching Engine and Two Towers builtin algorithm](get_started_with_matching_engine_twotowers.ipynb)
+
+```
+The steps performed include:
+1. Train the `Two-Tower` algorithm to generate embeddings (encoder) for the dataset.
+2. Hyperparameter tune the trained `Two-Tower` encoder.
+3. Make example predictions (embeddings) from then trained encoder.
+4. Generate embeddings using the trained `Two-Tower` builtin algorithm.
+5. Store embeddings to format supported by `Matching Engine`.
+6. Create a `Matching Engine Index` for the embeddings.
+7. Deploy the `Matching Engine Index` to a `Index Endpoint`.
+8. Make a matching engine prediction request.
+
+```
+
+[Get started with Optimized TensorFlow Enterprise container with Vertex AI Prediction / text models](get_started_with_optimized_tfe_bert.ipynb)
+
+```
+The steps performed include:
+- Download a pretrained BERT model from TensorFlow Hub.
+- Fine-tune (transfer learning) the BERT model as a binary classifier.
+- Upload the TensorFlow Hub model as a `Vertex AI Model` resource, with standard TensorFlow serving container.
+- Upload the TensorFlow Hub model as a `Vertex AI Model` resource, with TensorFlow Enterprise Optimized container
+- Create two `Endpoint` resources.
+- Deploying both `Model` resources to separate `Endpoint` resources.
+- Make the same online prediction requests to both `Model` resource instances deployed to the `Endpoint` resources.
+- Compare the prediction accuracy between the two deployed `Model` resources.
+- Configuring container settings for fine-tune control of optimizations.
+- Create a `Private Endpoint` resource.
+- Deploy the `Model` resoure with then `TensorFlow Enterprise Optimized` to the `Private Endpoint` resource.
+- Make an online prediction request to the `Private Endpoint` resource.
+
+```
+
+[Get started with Vertex AI Matching Engine](get_started_with_matching_engine.ipynb)
+
+```
+The steps performed include:
+- Create ANN Index.
+- Create an IndexEndpoint with VPC Network
+- Deploy ANN Index
+- Perform online query
+- Deploy brute force Index.
+- Perform calibration between ANN and brute force index.
+
+```
+
+[Get started with Vertex AI Matching Engine and Swivel builtin algorithm](get_started_with_matching_engine_swivel.ipynb)
+
+```
+The steps performed include:
+1. Train the `Swivel` algorithm to generate embeddings (encoder) for the dataset.
+2. Make example predictions (embeddings) from then trained encoder.
+3. Generate embeddings using the trained `Swivel` builtin algorithm.
+4. Store embeddings to format supported by `Matching Engine`.
+5. Create a `Matching Engine Index` for the embeddings.
+6. Deploy the `Matching Engine Index` to a `Index Endpoint`.
+7. Make a matching engine prediction request.
+
+```
+
+[Get started with TensorFlow serving with Vertex AI Prediction](get_started_with_tf_serving.ipynb)
+
+```
+The steps performed include:
 - Download a pretrained image classification model from TensorFlow Hub.
-- Upload the TensorFlow Hub model as a `Vertex AI Model` resource.
-- Make batch prediction with raw (uncompressed) image data to the `Model` resource, in JSONL format.
 - Create a serving function to receive compressed image data, and output decomopressed preprocessed data for the model input.
 - Upload the TensorFlow Hub model and serving function as a `Vertex AI Model` resource.
-- Make batch prediction with compressed image data to the `Model` resource, in File-List format.
-
-[Get started with Vertex AI Batch Prediction for AutoML video models](community/ml_ops/stage6/get_started_with_automl_video_model_batch.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Batch Prediction`  with a `AutoML` video model.
-
-The steps performed include:
-
-- Create a Vertex `Dataset` resource.
-- Train an `AutoML` model.
-- Make a batch prediction with JSONL input.
+- Creating an `Endpoint` resource.
+- Deploying the `Model` resource to an `Endpoint` resource with `TensorFlow Serving` serving binary.
+- Make an online prediction to the `Model` resource instance deployed to the `Endpoint` resource.
+```

notebooks/community/ml_ops/stage6/get_started_automl_tabular_exported_deploy.ipynb

@@ -0,0 +1,879 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "copyright"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2022 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "title:generic,gcp"
+      },
+      "source": [
+        "# E2E ML on GCP: MLOps stage 6 : serving: get started with re-importing AutoML tabular models\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_automl_tabular_exported_deploy.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_automl_tabular_exported_deploy.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage6/get_started_automl_tabular_exported_deploy.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>\n",
+        "<br/><br/><br/>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "overview:mlops"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "\n",
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with AutoML Training."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "objective:mlops,stage2,get_started_automl_training"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn how to use `AutoML Tabular` for re-importing exported model artifacts as a `Model` resource. This is useful for example, if one wants to move the exported model across projects.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `AutoML Tabular`\n",
+        "- `Vertex AI Model` resource\n",
+        "- `Vertex AI Prediction`\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Importing a pretrained AutoML tabular exported model artifacts, as a `Model` resource.\n",
+        "- Create an `Endpoint` resource.\n",
+        "- Deploy the `Model` resource to the `Endpoint` resource.\n",
+        "- Make a prediction."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:flowers,icn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "This tutorial uses a pretrained AutoML tabular model with exported model artifacts.\n",
+        "\n",
+        "\n",
+        "The tabular dataset used for the pretrained model is the GSOD dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). The version of the dataset you use only the fields year, month and day to predict the value of mean daily temperature (mean_temp).\n",
+        "\n",
+        "*Note:* This version of the exported model contains the custom op and requires the model server: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fb3451ce8e47"
+      },
+      "source": [
+        "### Costs\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "- Vertex AI\n",
+        "- Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "install_mlops"
+      },
+      "source": [
+        "## Installations\n",
+        "\n",
+        "Install the packages required for executing this notebook."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_mlops"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "# Install the packages\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
+        "! pip3 install --upgrade google-cloud-storage {USER_FLAG} -q"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "restart"
+      },
+      "source": [
+        "### Restart the kernel\n",
+        "\n",
+        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "restart"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "project_id"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "1. [Enable the Vertex AI, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage_component).\n",
+        "\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "56d591439df1"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_project_id"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "autoset_project_id"
+      },
+      "outputs": [],
+      "source": [
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID:\", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_gcloud_project_id"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud config set project $PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
+        "\n",
+        "- Americas: `us-central1`\n",
+        "- Europe: `europe-west4`\n",
+        "- Asia Pacific: `asia-east1`\n",
+        "\n",
+        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
+        "\n",
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "region"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "timestamp"
+      },
+      "source": [
+        "#### Timestamp\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "timestamp"
+      },
+      "outputs": [],
+      "source": [
+        "from datetime import datetime\n",
+        "\n",
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3ffa6b6c7cdb"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "\n",
+        "1. **Click Create service account**.\n",
+        "\n",
+        "2. In the **Service account name** field, enter a name, and click **Create**.\n",
+        "\n",
+        "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "\n",
+        "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2b72272258fc"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bucket:mbsdk"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
+        "\n",
+        "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bucket"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "autoset_bucket"
+      },
+      "outputs": [],
+      "source": [
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_bucket"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "create_bucket"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "validate_bucket"
+      },
+      "source": [
+        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "validate_bucket"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls -al $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "setup_vars"
+      },
+      "source": [
+        "### Set up variables\n",
+        "\n",
+        "Next, set up some variables used throughout the tutorial.\n",
+        "### Import libraries and define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_aip:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "import google.cloud.aiplatform as aiplatform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk"
+      },
+      "source": [
+        "### Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "init_aip:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "machine:training"
+      },
+      "source": [
+        "#### Set machine type\n",
+        "\n",
+        "Next, set the machine type to use for training.\n",
+        "\n",
+        "- Set the variable `DEPLOY_COMPUTE` to configure  the compute resources for the VMs you will use for for training.\n",
+        " - `machine type`\n",
+        "     - `n1-standard`: 3.75GB of memory per vCPU.\n",
+        "     - `n1-highmem`: 6.5GB of memory per vCPU\n",
+        "     - `n1-highcpu`: 0.9 GB of memory per vCPU\n",
+        " - `vCPUs`: number of \\[2, 4, 8, 16, 32, 64, 96 \\]\n",
+        "\n",
+        "*Note: The following is not supported for training:*\n",
+        "\n",
+        " - `standard`: 2 vCPUs\n",
+        " - `highcpu`: 2, 4 and 8 vCPUs\n",
+        "\n",
+        "*Note: You may also use n2 and e2 machine types for training and deployment, but they do not support GPUs*."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "machine:training"
+      },
+      "outputs": [],
+      "source": [
+        "if os.getenv(\"IS_TESTING_DEPLOY_MACHINE\"):\n",
+        "    MACHINE_TYPE = os.getenv(\"IS_TESTING_DEPLOY_MACHINE\")\n",
+        "else:\n",
+        "    MACHINE_TYPE = \"n1-standard\"\n",
+        "\n",
+        "VCPU = \"4\"\n",
+        "DEPLOY_COMPUTE = MACHINE_TYPE + \"-\" + VCPU\n",
+        "print(\"Train machine type\", DEPLOY_COMPUTE)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "import_file:u_dataset,csv"
+      },
+      "source": [
+        "### Location of pretrained `AutoML Tabular` exported model\n",
+        "\n",
+        "Now set the variable `MODEL_PACKAGE` to the location of the exported model artifacts in Cloud Storage."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "import_file:flowers,csv,icn"
+      },
+      "outputs": [],
+      "source": [
+        "MODEL_PACKAGE = \"gs://cloud-samples-data/vertex-ai/tabular-workflows/models/custom_op\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "quick_peek:csv"
+      },
+      "source": [
+        "#### Quick peek at your model package.\n",
+        "\n",
+        "Next, take a look at the contents of the model package for the exported AutoML tabular model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "quick_peek:csv"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil ls {MODEL_PACKAGE}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "automl_tabular_intro"
+      },
+      "source": [
+        "## AutoML tabular models\n",
+        "\n",
+        "AutoML can train the following types of tabular models:\n",
+        "\n",
+        "- classification\n",
+        "- regression\n",
+        "- forecasting\n",
+        "\n",
+        "A model can be trained for either automatic deployment to the cloud or exported for manual deployment to the cloud. In this tutorial, you use a pretrained exported AutoML tabular model.\n",
+        "\n",
+        "Learn more about [AutoML Model Types](https://cloud.google.com/vertex-ai/docs/start/automl-model-types)\n",
+        "\n",
+        "Learn more about [Exporting AutoML Tabular models](https://cloud.google.com/vertex-ai/docs/export/export-model-tabular)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c10efb34321b"
+      },
+      "source": [
+        "### Set the model server\n",
+        "\n",
+        "Next, you set the pre-built container for the model server. The container will be a version of `us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server`. If the model package contains an `environment.json` file, use the container version specified by the key `container_uri`; otherwise, use `us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:latest` "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a5f271de9040"
+      },
+      "outputs": [],
+      "source": [
+        "import json\n",
+        "\n",
+        "output = !gsutil cat {MODEL_PACKAGE}/environment.json\n",
+        "\n",
+        "MODEL_SERVER = json.loads(output[0])[\"container_uri\"]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e8ce91147c93"
+      },
+      "source": [
+        "### Upload the pretrained exported `AutoML Tabular` model package to a `Vertex AI Model` resource\n",
+        "\n",
+        "Next, you upload the model artifacts for the pretrained exported `AutoML Tabular` model into a `Vertex AI Model` resource, using the `Model.upload()` method with the following parameters:\n",
+        "\n",
+        "- `display_name`: A human readable name for the `Model` resource.\n",
+        "- `artifact_uri`: The Cloud Storage location of the model package.\n",
+        "- `serving_container_image_uri`: The serving container image.\n",
+        "- `serving_container_ports`: The serving port.\n",
+        "\n",
+        "*Note:* When you upload the model artifacts to a `Vertex Model` resource, you specify the corresponding deployment container image."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7988eae27f80"
+      },
+      "outputs": [],
+      "source": [
+        "model = aiplatform.Model.upload(\n",
+        "    display_name=\"gsod_\" + TIMESTAMP,\n",
+        "    artifact_uri=MODEL_PACKAGE,\n",
+        "    serving_container_image_uri=MODEL_SERVER,\n",
+        "    serving_container_ports=[8080],\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "628de0914ba1"
+      },
+      "source": [
+        "## Creating an `Endpoint` resource\n",
+        "\n",
+        "You create an `Endpoint` resource using the `Endpoint.create()` method. At a minimum, you specify the display name for the endpoint. Optionally, you can specify the project and location (region); otherwise the settings are inherited by the values you set when you initialized the Vertex AI SDK with the `init()` method.\n",
+        "\n",
+        "In this example, the following parameters are specified:\n",
+        "\n",
+        "- `display_name`: A human readable name for the `Endpoint` resource.\n",
+        "- `project`: Your project ID.\n",
+        "- `location`: Your region.\n",
+        "- `labels`: (optional) User defined metadata for the `Endpoint` in the form of key/value pairs.\n",
+        "\n",
+        "This method returns an `Endpoint` object.\n",
+        "\n",
+        "Learn more about [Vertex AI Endpoints](https://cloud.google.com/vertex-ai/docs/predictions/deploy-model-api)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0ea443f9593b"
+      },
+      "outputs": [],
+      "source": [
+        "endpoint = aiplatform.Endpoint.create(\n",
+        "    display_name=\"gsod_\" + TIMESTAMP,\n",
+        "    project=PROJECT_ID,\n",
+        "    location=REGION,\n",
+        "    labels={\"your_key\": \"your_value\"},\n",
+        ")\n",
+        "\n",
+        "print(endpoint)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ca3fa3f6a894"
+      },
+      "source": [
+        "## Deploying `Model` resources to an `Endpoint` resource.\n",
+        "\n",
+        "You can deploy one of more `Vertex AI Model` resource instances to the same endpoint. Each `Vertex AI Model` resource that is deployed will have its own deployment container for the serving binary. \n",
+        "\n",
+        "*Note:* For this example, you specified the deployment container for the exported AutoML Tabular model in the previous step of uploading the model artifacts to a `Vertex AI Model` resource.\n",
+        "\n",
+        "To deploy, you specify the following additional configuration settings:\n",
+        "\n",
+        "- The machine type.\n",
+        "- The (if any) type and number of GPUs.\n",
+        "- Static, manual or auto-scaling of VM instances.\n",
+        "\n",
+        "In this example, you deploy the model with the minimal amount of specified parameters, as follows:\n",
+        "\n",
+        "- `model`: The `Model` resource.\n",
+        "- `deployed_model_displayed_name`: The human readable name for the deployed model instance.\n",
+        "- `machine_type`: The machine type for each VM instance.\n",
+        "\n",
+        "Do to the requirements to provision the resource, this may take upto a few minutes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4e93b034a72f"
+      },
+      "outputs": [],
+      "source": [
+        "response = endpoint.deploy(\n",
+        "    model=model,\n",
+        "    deployed_model_display_name=\"gsod_\" + TIMESTAMP,\n",
+        "    machine_type=DEPLOY_COMPUTE,\n",
+        ")\n",
+        "\n",
+        "print(response)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f75331c946d5"
+      },
+      "source": [
+        "## Make a prediction\n",
+        "\n",
+        "Finally, you make an online prediction using the `endpoint()` method, with the following parameters:\n",
+        "\n",
+        "- `instances`: The instances to predict.\n",
+        "\n",
+        "The following is the for a prediction request:\n",
+        "\n",
+        "    [ INSTANCE_1, INSTANCE_2, ... ]\n",
+        "    \n",
+        "    INSTANCE : { \"column_1\": value, \"column_2\": value, ... }\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "72683bd9d777"
+      },
+      "outputs": [],
+      "source": [
+        "INSTANCES = [{\"year\": \"2020\", \"month\": \"1\", \"day\": \"23\"}]\n",
+        "\n",
+        "prediction = endpoint.predict(instances=INSTANCES)\n",
+        "print(prediction)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "endpoint_delete:mbsdk"
+      },
+      "source": [
+        "#### Delete the endpoint\n",
+        "\n",
+        "The method 'delete()' will delete the endpoint."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "endpoint_delete:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "endpoint.undeploy_all()\n",
+        "endpoint.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "model_delete:mbsdk"
+      },
+      "source": [
+        "#### Delete the model\n",
+        "\n",
+        "The method 'delete()' will delete the model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "model_delete:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "model.delete()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cleanup"
+      },
+      "source": [
+        "# Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cleanup"
+      },
+      "outputs": [],
+      "source": [
+        "# Set this to true only if you'd like to delete your bucket\n",
+        "delete_bucket = False\n",
+        "\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil rm -r $BUCKET_URI"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "get_started_automl_tabular_exported_deploy.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_batch.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2022 Google LLC\n",
+        "# Copyright 2021 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -33,20 +33,19 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_batch.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community/ml_ops/get_started_with_automl_image_model_batch.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_batch.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community/ml_ops/get_started_with_automl_image_model_batch.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_batch.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage6/get_started_with_automl_tabular_model_batch.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",
@@ -62,7 +61,7 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK to create image classification models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
+        "This tutorial demonstrates how to use the Vertex SDK to create image classification models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
       ]
     },
     {
@@ -73,7 +72,7 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to create an AutoML image classification model from a Python script, and then do a batch prediction using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
+        "In this tutorial, you create an AutoML image classification model from a Python script, and then do a batch prediction using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
         "\n",
         "This tutorial uses the following Google Cloud ML services and resources:\n",
         "\n",
@@ -245,7 +244,7 @@
         "\n",
         "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
         "\n",
-        "4. If you are running this notebook locally, you need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
+        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
         "\n",
         "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
@@ -253,17 +252,6 @@
         "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "project_id"
-      },
-      "source": [
-        "#### Set your project ID\n",
-        "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -647,7 +635,7 @@
         "dataset = aiplatform.ImageDataset.create(\n",
         "    display_name=\"Flowers\" + \"_\" + UUID,\n",
         "    gcs_source=[IMPORT_FILE],\n",
-        "    import_schema_uri=aiplatform.schema.dataset.ioformat.image.single_label_classification,\n",
+        "    import_schema_uri=aip.schema.dataset.ioformat.image.single_label_classification,\n",
         ")\n",
         "\n",
         "print(dataset.resource_name)"
@@ -723,7 +711,7 @@
         "\n",
         "The `run` method when completed returns the `Model` resource.\n",
         "\n",
-        "The execution of the training pipeline may take upto 20 minutes."
+        "The execution of the training pipeline will take upto 20 minutes."
       ]
     },
     {
@@ -755,7 +743,7 @@
         "\n",
         "Batch prediction provides the ability to do offline batch processing of large amounts of prediction requests. Resources are only provisioned during the batch process and then deprovisioned when the batch request is completed. The results are stored in Cloud Storage, in contrast to online prediction where the results are returned as a HTTP response packet.\n",
         "\n",
-        "The input format for your batch job is dependent on the format supported by your model server. Foremost, the web server in your model server must support a JSONL format, which the web server converts to a format support either directly by the model input intertace or a serving function interface. For batch prediction, this JSONL format is referred to as the `pivot` format.\n",
+        "The input format for your batch job is dependent on the format supported by your model server. Foremost, the web server in your model server must support a JSONL format, which the web server will convert to a format support either directly by the model input intertace or a serving function interface. For batch prediction, this JSONL format is referred to as the `pivot` format.\n",
         "\n",
         "### Input format for batch prediction jobs\n",
         "\n",
@@ -764,7 +752,7 @@
         "\n",
         "- JSONL\n",
         "\n",
-        "The batch server accepts the following output formats for AutoML image models:\n",
+        "The batch server accepts the following input formats for AutoML image models:\n",
         "\n",
         "- JSONL\n",
         "\n",
@@ -788,7 +776,7 @@
         "\n",
         "**CSV**\n",
         "\n",
-        "The csv header in the first line is always be ignored. String fields are required to be double quoted explicitly, otherwise the row is discarded and parsing error messages are outputted to error files. Non-quoted values are always transferred as floats.\n",
+        "The csv header in the first line will always be ignored. String fields are required to be double quoted explicitly, otherwise the row is discarded and parsing error messages are outputted to error files. Non-quoted values are always transferred as floats.\n",
         "\n",
         "    col1,col2,col3\n",
         "    1,3,\"cat1\"\n",
@@ -966,7 +954,7 @@
         "- `prediction_format`: The format of the batch prediction response file: \"jsonl\", \"csv\", \"bigquery\", \"tf-record\", \"tf-record-gzip\" or \"file-list\"\n",
         "- `gcs_source`: A list of one or more batch request input files.\n",
         "- `gcs_destination_prefix`: The Cloud Storage location for storing the batch prediction resuls.\n",
-        "- `sync`: If set to True, the call blocks while waiting for the asynchronous batch job to complete."
+        "- `sync`: If set to True, the call will block while waiting for the asynchronous batch job to complete."
       ]
     },
     {

notebooks/community/ml_ops/stage6/get_started_with_automl_tabular_exported_deploy.ipynb

@@ -1,898 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "copyright"
-   },
-   "outputs": [],
-   "source": [
-    "# Copyright 2022 Google LLC\n",
-    "#\n",
-    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-    "# you may not use this file except in compliance with the License.\n",
-    "# You may obtain a copy of the License at\n",
-    "#\n",
-    "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-    "#\n",
-    "# Unless required by applicable law or agreed to in writing, software\n",
-    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-    "# See the License for the specific language governing permissions and\n",
-    "# limitations under the License."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "title:generic,gcp"
-   },
-   "source": [
-    "# E2E ML on GCP: MLOps stage 6 : serving: get started with re-importing AutoML tabular models\n",
-    "\n",
-    "<table align=\"left\">\n",
-    "  <td>\n",
-    "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_automl_tabular_exported_deploy.ipynb\">\n",
-    "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-    "      View on GitHub\n",
-    "    </a>\n",
-    "  </td>\n",
-    "  <td>\n",
-    "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_automl_with_tabular_exported_deploy.ipynb\">\n",
-    "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-    "        </a>\n",
-    "  </td>\n",
-    "  <td>\n",
-    "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage6/get_started_automl_with_tabular_exported_deploy.ipynb\">\n",
-    "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-    "      Open in Vertex AI Workbench\n",
-    "    </a>\n",
-    "  </td>\n",
-    "</table>\n",
-    "<br/><br/><br/>"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "overview:mlops"
-   },
-   "source": [
-    "## Overview\n",
-    "\n",
-    "\n",
-    "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with AutoML Training."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "objective:mlops,stage2,get_started_automl_training"
-   },
-   "source": [
-    "### Objective\n",
-    "\n",
-    "In this tutorial, you learn how to use `AutoML Tabular` for re-importing exported model artifacts as a `Model` resource. This is useful for example, if one wants to move the exported model across projects.\n",
-    "\n",
-    "This tutorial uses the following Google Cloud ML services:\n",
-    "\n",
-    "- `AutoML Tabular`\n",
-    "- `Vertex AI Model` resource\n",
-    "- `Vertex AI Prediction`\n",
-    "\n",
-    "The steps performed include:\n",
-    "\n",
-    "- Importing a pretrained AutoML tabular exported model artifacts, as a `Model` resource.\n",
-    "- Create an `Endpoint` resource.\n",
-    "- Deploy the `Model` resource to the `Endpoint` resource.\n",
-    "- Make a prediction."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "dataset:flowers,icn"
-   },
-   "source": [
-    "### Dataset\n",
-    "\n",
-    "This tutorial uses a pretrained AutoML tabular model with exported model artifacts.\n",
-    "\n",
-    "\n",
-    "The tabular dataset used for the pretrained model is the GSOD dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). The version of the dataset you use only the fields year, month and day to predict the value of mean daily temperature (mean_temp).\n",
-    "\n",
-    "*Note:* This version of the exported model contains the custom op and requires the model server: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "fb3451ce8e47"
-   },
-   "source": [
-    "### Costs\n",
-    "This tutorial uses billable components of Google Cloud:\n",
-    "\n",
-    "- Vertex AI\n",
-    "- Cloud Storage\n",
-    "\n",
-    "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing) and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "install_mlops"
-   },
-   "source": [
-    "## Installations\n",
-    "\n",
-    "Install the packages required for executing this notebook."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "install_mlops"
-   },
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "\n",
-    "# The Vertex AI Workbench Notebook product has specific requirements\n",
-    "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
-    "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-    "    \"/opt/deeplearning/metadata/env_version\"\n",
-    ")\n",
-    "\n",
-    "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-    "USER_FLAG = \"\"\n",
-    "if IS_WORKBENCH_NOTEBOOK:\n",
-    "    USER_FLAG = \"--user\"\n",
-    "\n",
-    "# Install the packages\n",
-    "\n",
-    "! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
-    "! pip3 install --upgrade google-cloud-storage {USER_FLAG} -q"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "restart"
-   },
-   "source": [
-    "### Restart the kernel\n",
-    "\n",
-    "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "restart"
-   },
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "\n",
-    "if not os.getenv(\"IS_TESTING\"):\n",
-    "    # Automatically restart kernel after installs\n",
-    "    import IPython\n",
-    "\n",
-    "    app = IPython.Application.instance()\n",
-    "    app.kernel.do_shutdown(True)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "project_id"
-   },
-   "source": [
-    "## Before you begin\n",
-    "\n",
-    "### Set up your Google Cloud project\n",
-    "\n",
-    "**The following steps are required, regardless of your notebook environment.**\n",
-    "\n",
-    "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-    "\n",
-    "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-    "\n",
-    "1. [Enable the Vertex AI, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,storage_component).\n",
-    "\n",
-    "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
-    "\n",
-    "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-    "Cloud SDK uses the right project for all the commands in this notebook.\n",
-    "\n",
-    "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "56d591439df1"
-   },
-   "source": [
-    "#### Set your project ID\n",
-    "\n",
-    "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "set_project_id"
-   },
-   "outputs": [],
-   "source": [
-    "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "autoset_project_id"
-   },
-   "outputs": [],
-   "source": [
-    "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-    "    # Get your GCP project id from gcloud\n",
-    "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-    "    PROJECT_ID = shell_output[0]\n",
-    "    print(\"Project ID:\", PROJECT_ID)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "set_gcloud_project_id"
-   },
-   "outputs": [],
-   "source": [
-    "! gcloud config set project $PROJECT_ID"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "region"
-   },
-   "source": [
-    "#### Region\n",
-    "\n",
-    "You can also change the `REGION` variable, which is used for operations\n",
-    "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
-    "\n",
-    "- Americas: `us-central1`\n",
-    "- Europe: `europe-west4`\n",
-    "- Asia Pacific: `asia-east1`\n",
-    "\n",
-    "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-    "\n",
-    "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "region"
-   },
-   "outputs": [],
-   "source": [
-    "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-    "\n",
-    "if REGION == \"[your-region]\":\n",
-    "    REGION = \"us-central1\""
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "timestamp"
-   },
-   "source": [
-    "#### Timestamp\n",
-    "\n",
-    "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "timestamp"
-   },
-   "outputs": [],
-   "source": [
-    "from datetime import datetime\n",
-    "\n",
-    "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "3ffa6b6c7cdb"
-   },
-   "source": [
-    "### Authenticate your Google Cloud account\n",
-    "\n",
-    "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
-    "\n",
-    "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
-    "\n",
-    "**Otherwise**, follow these steps:\n",
-    "\n",
-    "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
-    "\n",
-    "1. **Click Create service account**.\n",
-    "\n",
-    "2. In the **Service account name** field, enter a name, and click **Create**.\n",
-    "\n",
-    "3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-    "\n",
-    "4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
-    "\n",
-    "5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "2b72272258fc"
-   },
-   "outputs": [],
-   "source": [
-    "# If you are running this notebook in Colab, run this cell and follow the\n",
-    "# instructions to authenticate your GCP account. This provides access to your\n",
-    "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
-    "# requests.\n",
-    "\n",
-    "import os\n",
-    "import sys\n",
-    "\n",
-    "# If on Vertex AI Workbench, then don't execute this code\n",
-    "IS_COLAB = \"google.colab\" in sys.modules\n",
-    "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-    "    \"DL_ANACONDA_HOME\"\n",
-    "):\n",
-    "    if \"google.colab\" in sys.modules:\n",
-    "        from google.colab import auth as google_auth\n",
-    "\n",
-    "        google_auth.authenticate_user()\n",
-    "\n",
-    "    # If you are running this notebook locally, replace the string below with the\n",
-    "    # path to your service account key and run this cell to authenticate your GCP\n",
-    "    # account.\n",
-    "    elif not os.getenv(\"IS_TESTING\"):\n",
-    "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "bucket:mbsdk"
-   },
-   "source": [
-    "### Create a Cloud Storage bucket\n",
-    "\n",
-    "**The following steps are required, regardless of your notebook environment.**\n",
-    "\n",
-    "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
-    "\n",
-    "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "bucket"
-   },
-   "outputs": [],
-   "source": [
-    "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-    "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "autoset_bucket"
-   },
-   "outputs": [],
-   "source": [
-    "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-    "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-    "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "create_bucket"
-   },
-   "source": [
-    "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "create_bucket"
-   },
-   "outputs": [],
-   "source": [
-    "! gsutil mb -l $REGION $BUCKET_URI"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "validate_bucket"
-   },
-   "source": [
-    "Finally, validate access to your Cloud Storage bucket by examining its contents:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "validate_bucket"
-   },
-   "outputs": [],
-   "source": [
-    "! gsutil ls -al $BUCKET_URI"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "setup_vars"
-   },
-   "source": [
-    "### Set up variables\n",
-    "\n",
-    "Next, set up some variables used throughout the tutorial.\n",
-    "### Import libraries and define constants"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "import_aip:mbsdk"
-   },
-   "outputs": [],
-   "source": [
-    "import google.cloud.aiplatform as aiplatform"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "init_aip:mbsdk"
-   },
-   "source": [
-    "### Initialize Vertex AI SDK for Python\n",
-    "\n",
-    "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "init_aip:mbsdk"
-   },
-   "outputs": [],
-   "source": [
-    "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "machine:training"
-   },
-   "source": [
-    "#### Set machine type\n",
-    "\n",
-    "Next, set the machine type to use for training.\n",
-    "\n",
-    "- Set the variable `DEPLOY_COMPUTE` to configure  the compute resources for the VMs you will use for for training.\n",
-    " - `machine type`\n",
-    "     - `n1-standard`: 3.75GB of memory per vCPU.\n",
-    "     - `n1-highmem`: 6.5GB of memory per vCPU\n",
-    "     - `n1-highcpu`: 0.9 GB of memory per vCPU\n",
-    " - `vCPUs`: number of \\[2, 4, 8, 16, 32, 64, 96 \\]\n",
-    "\n",
-    "*Note: The following is not supported for training:*\n",
-    "\n",
-    " - `standard`: 2 vCPUs\n",
-    " - `highcpu`: 2, 4 and 8 vCPUs\n",
-    "\n",
-    "*Note: You may also use n2 and e2 machine types for training and deployment, but they do not support GPUs*."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "machine:training"
-   },
-   "outputs": [],
-   "source": [
-    "if os.getenv(\"IS_TESTING_DEPLOY_MACHINE\"):\n",
-    "    MACHINE_TYPE = os.getenv(\"IS_TESTING_DEPLOY_MACHINE\")\n",
-    "else:\n",
-    "    MACHINE_TYPE = \"n1-standard\"\n",
-    "\n",
-    "VCPU = \"4\"\n",
-    "DEPLOY_COMPUTE = MACHINE_TYPE + \"-\" + VCPU\n",
-    "print(\"Train machine type\", DEPLOY_COMPUTE)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "import_file:u_dataset,csv"
-   },
-   "source": [
-    "### Location of pretrained `AutoML Tabular` exported model\n",
-    "\n",
-    "Now set the variable `MODEL_PACKAGE` to the location of the exported model artifacts in Cloud Storage."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "import_file:flowers,csv,icn"
-   },
-   "outputs": [],
-   "source": [
-    "MODEL_PACKAGE = \"gs://cloud-samples-data/vertex-ai/tabular-workflows/models/custom_op\""
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "quick_peek:csv"
-   },
-   "source": [
-    "#### Quick peek at your model package.\n",
-    "\n",
-    "Next, take a look at the contents of the model package for the exported AutoML tabular model."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "quick_peek:csv"
-   },
-   "outputs": [],
-   "source": [
-    "! gsutil ls {MODEL_PACKAGE}"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "automl_tabular_intro"
-   },
-   "source": [
-    "## AutoML tabular models\n",
-    "\n",
-    "AutoML can train the following types of tabular models:\n",
-    "\n",
-    "- classification\n",
-    "- regression\n",
-    "- forecasting\n",
-    "\n",
-    "A model can be trained for either automatic deployment to the cloud or exported for manual deployment to the cloud. In this tutorial, you use a pretrained exported AutoML tabular model.\n",
-    "\n",
-    "Learn more about [AutoML Model Types](https://cloud.google.com/vertex-ai/docs/start/automl-model-types)\n",
-    "\n",
-    "Learn more about [Exporting AutoML Tabular models](https://cloud.google.com/vertex-ai/docs/export/export-model-tabular)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "c10efb34321b"
-   },
-   "source": [
-    "### Set the model server\n",
-    "\n",
-    "Next, you set the pre-built container for the model server. The container will be a version of `us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server`. If the model package contains an `environment.json` file, use the container version specified by the key `container_uri`; otherwise, use `us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:latest` "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "a5f271de9040"
-   },
-   "outputs": [],
-   "source": [
-    "import json\n",
-    "\n",
-    "output = !gsutil cat {MODEL_PACKAGE}/environment.json\n",
-    "\n",
-    "MODEL_SERVER = json.loads(output[0])[\"container_uri\"]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "e8ce91147c93"
-   },
-   "source": [
-    "### Upload the pretrained exported `AutoML Tabular` model package to a `Vertex AI Model` resource\n",
-    "\n",
-    "Next, you upload the model artifacts for the pretrained exported `AutoML Tabular` model into a `Vertex AI Model` resource, using the `Model.upload()` method with the following parameters:\n",
-    "\n",
-    "- `display_name`: A human readable name for the `Model` resource.\n",
-    "- `artifact_uri`: The Cloud Storage location of the model package.\n",
-    "- `serving_container_image_uri`: The serving container image.\n",
-    "- `serving_container_ports`: The serving port.\n",
-    "\n",
-    "*Note:* When you upload the model artifacts to a `Vertex Model` resource, you specify the corresponding deployment container image."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "7988eae27f80"
-   },
-   "outputs": [],
-   "source": [
-    "model = aiplatform.Model.upload(\n",
-    "    display_name=\"gsod_\" + TIMESTAMP,\n",
-    "    artifact_uri=MODEL_PACKAGE,\n",
-    "    serving_container_image_uri=MODEL_SERVER,\n",
-    "    serving_container_ports=[8080],\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "628de0914ba1"
-   },
-   "source": [
-    "## Creating an `Endpoint` resource\n",
-    "\n",
-    "You create an `Endpoint` resource using the `Endpoint.create()` method. At a minimum, you specify the display name for the endpoint. Optionally, you can specify the project and location (region); otherwise the settings are inherited by the values you set when you initialized the Vertex AI SDK with the `init()` method.\n",
-    "\n",
-    "In this example, the following parameters are specified:\n",
-    "\n",
-    "- `display_name`: A human readable name for the `Endpoint` resource.\n",
-    "- `project`: Your project ID.\n",
-    "- `location`: Your region.\n",
-    "- `labels`: (optional) User defined metadata for the `Endpoint` in the form of key/value pairs.\n",
-    "\n",
-    "This method returns an `Endpoint` object.\n",
-    "\n",
-    "Learn more about [Vertex AI Endpoints](https://cloud.google.com/vertex-ai/docs/predictions/deploy-model-api)."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "0ea443f9593b"
-   },
-   "outputs": [],
-   "source": [
-    "endpoint = aiplatform.Endpoint.create(\n",
-    "    display_name=\"gsod_\" + TIMESTAMP,\n",
-    "    project=PROJECT_ID,\n",
-    "    location=REGION,\n",
-    "    labels={\"your_key\": \"your_value\"},\n",
-    ")\n",
-    "\n",
-    "print(endpoint)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "ca3fa3f6a894"
-   },
-   "source": [
-    "## Deploying `Model` resources to an `Endpoint` resource.\n",
-    "\n",
-    "You can deploy one of more `Vertex AI Model` resource instances to the same endpoint. Each `Vertex AI Model` resource that is deployed will have its own deployment container for the serving binary. \n",
-    "\n",
-    "*Note:* For this example, you specified the deployment container for the exported AutoML Tabular model in the previous step of uploading the model artifacts to a `Vertex AI Model` resource.\n",
-    "\n",
-    "To deploy, you specify the following additional configuration settings:\n",
-    "\n",
-    "- The machine type.\n",
-    "- The (if any) type and number of GPUs.\n",
-    "- Static, manual or auto-scaling of VM instances.\n",
-    "\n",
-    "In this example, you deploy the model with the minimal amount of specified parameters, as follows:\n",
-    "\n",
-    "- `model`: The `Model` resource.\n",
-    "- `deployed_model_displayed_name`: The human readable name for the deployed model instance.\n",
-    "- `machine_type`: The machine type for each VM instance.\n",
-    "\n",
-    "Do to the requirements to provision the resource, this may take upto a few minutes."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "4e93b034a72f"
-   },
-   "outputs": [],
-   "source": [
-    "response = endpoint.deploy(\n",
-    "    model=model,\n",
-    "    deployed_model_display_name=\"gsod_\" + TIMESTAMP,\n",
-    "    machine_type=DEPLOY_COMPUTE,\n",
-    ")\n",
-    "\n",
-    "print(response)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "f75331c946d5"
-   },
-   "source": [
-    "## Make a prediction\n",
-    "\n",
-    "Finally, you make an online prediction using the `endpoint()` method, with the following parameters:\n",
-    "\n",
-    "- `instances`: The instances to predict.\n",
-    "\n",
-    "The following is the for a prediction request:\n",
-    "\n",
-    "    [ INSTANCE_1, INSTANCE_2, ... ]\n",
-    "    \n",
-    "    INSTANCE : { \"column_1\": value, \"column_2\": value, ... }\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "72683bd9d777"
-   },
-   "outputs": [],
-   "source": [
-    "INSTANCES = [{\"year\": \"2020\", \"month\": \"1\", \"day\": \"23\"}]\n",
-    "\n",
-    "prediction = endpoint.predict(instances=INSTANCES)\n",
-    "print(prediction)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "endpoint_delete:mbsdk"
-   },
-   "source": [
-    "#### Delete the endpoint\n",
-    "\n",
-    "The method 'delete()' will delete the endpoint."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "endpoint_delete:mbsdk"
-   },
-   "outputs": [],
-   "source": [
-    "endpoint.undeploy_all()\n",
-    "endpoint.delete()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "model_delete:mbsdk"
-   },
-   "source": [
-    "#### Delete the model\n",
-    "\n",
-    "The method 'delete()' will delete the model."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "model_delete:mbsdk"
-   },
-   "outputs": [],
-   "source": [
-    "model.delete()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "cleanup"
-   },
-   "source": [
-    "# Cleaning up\n",
-    "\n",
-    "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
-    "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
-    "\n",
-    "Otherwise, you can delete the individual resources you created in this tutorial.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "cleanup"
-   },
-   "outputs": [],
-   "source": [
-    "# Set this to true only if you'd like to delete your bucket\n",
-    "delete_bucket = False\n",
-    "\n",
-    "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-    "    ! gsutil rm -r $BUCKET_URI"
-   ]
-  }
- ],
- "metadata": {
-  "colab": {
-   "name": "get_started_automl_tabular_exported_deploy.ipynb",
-   "toc_visible": true
-  },
-  "environment": {
-   "kernel": "python3",
-   "name": "common-cpu.m95",
-   "type": "gcloud",
-   "uri": "gcr.io/deeplearning-platform-release/base-cpu:m95"
-  },
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.7.12"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}

notebooks/community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb

@@ -741,16 +741,13 @@
         "\n",
         "### Input format for batch prediction jobs\n",
         "\n",
-        "The batch server accepts the following input formats for custom image models:\n",
+        "The batch server accepts the following input formats:\n",
         "\n",
         "- JSONL\n",
+        "- CSV\n",
+        "- TFRecords\n",
         "- File-List\n",
-        "\n",
-        "### Output format for batch prediction jobs\n",
-        "\n",
-        "The batch server accepts the following output formats for custom image models:\n",
-        "\n",
-        "- JSONL\n",
+        "- BigQuery table\n",
         "\n",
         "### Pivot format\n",
         "\n",
@@ -1309,6 +1306,7 @@
       "source": [
         "### Send the prediction request\n",
         "\n",
+        "BLAH\n",
         "\n",
         "To make a batch prediction request, call the model object's `batch_predict` method with the following parameters: \n",
         "- `instances_format`: The format of the batch prediction request file: \"jsonl\", \"csv\", \"bigquery\", \"tf-record\", \"tf-record-gzip\" or \"file-list\"\n",

notebooks/community/ml_ops/stage6/get_started_with_custom_tabular_model_batch.ipynb

@@ -110,7 +110,7 @@
         "- identity - unique player identitity numbers\n",
         "- demographic features - information about the player, such as the geographic region in which a player is located\n",
         "- behavioral features - counts of the number of times a  player has triggered certain game events, such as reaching a new level\n",
-        "- churn propensity - this is the label or target feature, it provides an estimated probability that this player may churn, i.e. stop being an active player.\n",
+        "- churn propensity - this is the label or target feature, it provides an estimated probability that this player will churn, i.e. stop being an active player.\n",
         "\n",
         "**CSV batch input example**\n",
         "\n",
@@ -574,7 +574,7 @@
         "\n",
         "Learn more about [hardware accelerator support for your region](https://cloud.google.com/vertex-ai/docs/general/locations#accelerators).\n",
         "\n",
-        "*Note*: TF releases before 2.3 for GPU support fails to load the custom model in this tutorial. It is a known issue and fixed in TF 2.3. This is caused by static graph ops that are generated in the serving function. If you encounter this issue on your own custom models, use a container image for TF 2.3 with GPU support."
+        "*Note*: TF releases before 2.3 for GPU support will fail to load the custom model in this tutorial. It is a known issue and fixed in TF 2.3. This is caused by static graph ops that are generated in the serving function. If you encounter this issue on your own custom models, use a container image for TF 2.3 with GPU support."
       ]
     },
     {
@@ -638,7 +638,7 @@
         "\n",
         "Next, set the machine type to use for prediction.\n",
         "\n",
-        "- Set the variable `DEPLOY_COMPUTE` to configure  the compute resources for the VMs you use for for prediction.\n",
+        "- Set the variable `DEPLOY_COMPUTE` to configure  the compute resources for the VMs you will use for for prediction.\n",
         " - `machine type`\n",
         "     - `n1-standard`: 3.75GB of memory per vCPU.\n",
         "     - `n1-highmem`: 6.5GB of memory per vCPU\n",
@@ -712,7 +712,7 @@
         "\n",
         "Batch prediction provides the ability to do offline batch processing of large amounts of prediction requests. Resources are only provisioned during the batch process and then deprovisioned when the batch request is completed. The results are stored in Cloud Storage, in contrast to online prediction where the results are returned as a HTTP response packet.\n",
         "\n",
-        "The input format for your batch job is dependent on the format supported by your model server. Foremost, the web server in your model server must support a JSONL format, which the web server converts to a format support either directly by the model input intertace or a serving function interface. For batch prediction, this JSONL format is referred to as the `pivot` format.\n",
+        "The input format for your batch job is dependent on the format supported by your model server. Foremost, the web server in your model server must support a JSONL format, which the web server will convert to a format support either directly by the model input intertace or a serving function interface. For batch prediction, this JSONL format is referred to as the `pivot` format.\n",
         "\n",
         "### Input format for batch prediction jobs\n",
         "\n",
@@ -749,7 +749,7 @@
         "\n",
         "**CSV**\n",
         "\n",
-        "The csv header in the first line is always be ignored. String fields are required to be double quoted explicitly, otherwise the row is discarded and parsing error messages are outputted to error files. Non-quoted values are always transferred as floats.\n",
+        "The csv header in the first line will always be ignored. String fields are required to be double quoted explicitly, otherwise the row is discarded and parsing error messages are outputted to error files. Non-quoted values are always transferred as floats.\n",
         "\n",
         "    col1,col2,col3\n",
         "    1,3,\"cat1\"\n",
@@ -910,7 +910,7 @@
         "- `prediction_format`: The format of the batch prediction response file: \"jsonl\", \"csv\", \"bigquery\", \"tf-record\", \"tf-record-gzip\" or \"file-list\"\n",
         "- `job_display_name`: The human readable name for the prediction job.\n",
         " - `gcs_source`: A list of one or more Cloud Storage paths to your batch prediction requests.\n",
-        "- `gcs_destination_prefix`: The Cloud Storage path that the service writes the predictions to.\n",
+        "- `gcs_destination_prefix`: The Cloud Storage path that the service will write the predictions to.\n",
         "- `model_parameters`: Additional filtering parameters for serving prediction results.\n",
         "- `machine_type`: The type of machine to use for training.\n",
         "- `accelerator_type`: The hardware accelerator type.\n",

notebooks/community/ml_ops/stage6/get_started_with_nvidia_triton_serving.ipynb

@@ -81,7 +81,7 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to deploy a container running Nvidia Triton Server with a `Vertex AI Model` resource to a `Vertex AI Endpoint` for making online predictions.\n",
+        "In this tutorial, you deploy a container running Nvidia Triton Server with a `Vertex AI Model` resource to a `Vertex AI Endpoint` for making online predictions.\n",
         "\n",
         "This tutorial uses the following Google Cloud ML services:\n",
         "\n",

notebooks/community/ml_ops/stage7/README.md

@@ -34,69 +34,3 @@ This stage may be done entirely by MLOps. We recommend:
 ## Notebooks
 
 ### Get Started
-
-[Vertex AI Model Monitoring for custom tabular models with TensorFlow Serving container](community/ml_ops/stage7/get_started_with_model_monitoring_custom_tf_serving.ipynb)
-
-In this notebook, you learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests, for custom tabular models, using a custom deployment container. 
-
-The steps performed include:
-
-- Download a pre-trained custom tabular model.
-- Upload the pre-trained model as a `Model` resource.
-- Deploying the `Model` resource to an `Endpoint` resource with `TensorFlow Serving` serving binary.
-- Configure the `Endpoint` resource for model monitoring.
-- Generate synthetic prediction requests for skew.
-- Wait for email alert notification.
-- Generate synthetic prediction requests for drift.
-- Wait for email alert notification.
-
-
-
-[Vertex AI Model Monitoring for AutoML tabular models](community/ml_ops/stage7/get_started_with_model_monitoring_automl.ipynb)
-
-In this notebook, you learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests, for AutoML tabular models.
-
-The steps performed include:
-
-- Train an `AutoML` model.
-- Deploy the `Model` resource to the `Endpoint` resource.
-- Configure the `Endpoint` resource for model monitoring.
-- Generate synthetic prediction requests for skew.
-- Wait for email alert notification.
-- Generate synthetic prediction requests for drift.
-- Wait for email alert notification.
-
-
-[Vertex AI Model Monitoring for custom tabular models](community/ml_ops/stage7/get_started_with_model_monitoring_custom.ipynb)
-
-In this notebook, you learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests, for custom tabular models.
-
-The steps performed include:
-
-- Download a pre-trained custom tabular model.
-- Upload the pre-trained model as a `Model` resource.
-- Deploy the `Model` resource to the `Endpoint` resource.
-- Configure the `Endpoint` resource for model monitoring.
-- Generate synthetic prediction requests for skew.
-- Wait for email alert notification.
-- Generate synthetic prediction requests for drift.
-- Wait for email alert notification.
-
-
-
-[Vertex AI Model Monitoring for setup for tabular models](community/ml_ops/stage7/get_started_with_model_monitoring_setup.ipynb)
-
-In this notebook, you learn to setup the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests.
-
-The steps performed include:
-
-- Download a pre-trained custom tabular model.
-- Upload the pre-trained model as a `Model` resource.
-- Deploy the `Model` resource to the `Endpoint` resource.
-- Configure the `Endpoint` resource for model monitoring.
-    - Skew and drift detection for feature inputs.
-    - Skew and drift detection for feature attributions.
-- Automatic generation of the `input schema` by sending 1000 prediction request.
-- List, pause, resume and delete monitoring jobs.
-- Restart monitoring job with predefined `input schema`.
-- View logged monitored data.

notebooks/community/model_monitoring/batch_prediction_model_monitoring.ipynb

@@ -3,6 +3,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "d3069d95",
       "metadata": {
         "cellView": "form",
         "id": "d3069d95"
@@ -10,7 +11,7 @@
       "outputs": [],
       "source": [
         "# @title Copyright & License (click to expand)\n",
-        "# Copyright 2022 Google LLC\n",
+        "# Copyright 2021 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -27,6 +28,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "546c53de",
       "metadata": {
         "id": "546c53de"
       },
@@ -44,16 +46,13 @@
         "        <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
-        "  </td><td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai-platform/notebooks/deploy-notebook?name=Model%20Monitoring&download_url=https%3A%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_monitoring%2Fbatch_prediction_model_monitoring.ipynb\">\n",
-        "       <img src=\"https://www.gstatic.com/cloud/images/navigation/vertex-ai.svg\" alt=\"Google Cloud Notebooks\">Open in Workbench AI Notebook\n",
-        "    </a>\n",
-        "  </td> \n",
+        "  </td>\n",
         "</table>"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "53fd1070",
       "metadata": {
         "id": "53fd1070"
       },
@@ -65,6 +64,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "8b26c855",
       "metadata": {
         "id": "8b26c855"
       },
@@ -98,6 +98,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "d52ba95b",
       "metadata": {
         "id": "d52ba95b"
       },
@@ -109,6 +110,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "e64fb18a",
       "metadata": {
         "id": "e64fb18a"
       },
@@ -121,6 +123,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "9d839347",
       "metadata": {
         "id": "9d839347"
       },
@@ -139,6 +142,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "738fce1f",
       "metadata": {
         "id": "738fce1f"
       },
@@ -151,6 +155,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "4536fe4e",
       "metadata": {
         "id": "4536fe4e"
       },
@@ -173,13 +178,14 @@
         "    USER_FLAG = \"--user\"\n",
         "\n",
         "# Install Python package dependencies.\n",
-        "! pip3 install -q {USER_FLAG} tensorflow-data-validation \\\n",
-        "                              google-api-core \\\n",
-        "                              google-cloud-aiplatform"
+        "! pip3 install -q tensorflow-data-validation $USER_FLAG\n",
+        "! pip3 install -q google-api-core $USER_FLAG\n",
+        "! pip3 install -q google-cloud-aiplatform $USER_FLAG"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "6e98402b",
       "metadata": {
         "id": "6e98402b"
       },
@@ -192,6 +198,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "9775c9ff",
       "metadata": {
         "id": "9775c9ff"
       },
@@ -210,16 +217,13 @@
     },
     {
       "cell_type": "markdown",
+      "id": "d5737134",
       "metadata": {
         "id": "d5737134"
       },
       "source": [
         "### Set up your Google Cloud project\n",
         "\n",
-        "**_NOTE_**: This notebook has been tested in the following environment:\n",
-        "\n",
-        "* Python version = 3.7\n",
-        "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
         "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
@@ -238,6 +242,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "cfb1a1d5",
       "metadata": {
         "id": "cfb1a1d5"
       },
@@ -250,33 +255,50 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "cf8535e4",
       "metadata": {
         "id": "cf8535e4"
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+        "import os\n",
+        "\n",
+        "PROJECT_ID = \"\"\n",
+        "\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID: \", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "05a2d397",
+      "metadata": {
+        "id": "05a2d397"
+      },
+      "source": [
+        "Otherwise, set your project ID here.\n"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "1c2be4bd",
       "metadata": {
         "id": "1c2be4bd"
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or not PROJECT_ID or PROJECT_ID == \"[your-project-id]\":\n",
-        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"python-docs-samples-tests\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "c129705c",
       "metadata": {
         "id": "c129705c"
       },
@@ -287,6 +309,32 @@
     },
     {
       "cell_type": "markdown",
+      "id": "71404c9f",
+      "metadata": {
+        "id": "71404c9f"
+      },
+      "source": [
+        "#### Set your email address\n",
+        "This is used for delivering model monitoring notifications.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "4b1d2b69",
+      "metadata": {
+        "id": "4b1d2b69"
+      },
+      "outputs": [],
+      "source": [
+        "EMAIL_ADDRESS = \"[your-email-address]\"  # @param {type:\"string\"}\n",
+        "if not EMAIL_ADDRESS or EMAIL_ADDRESS == \"[your-email-address]\":\n",
+        "    print(\"EMAIL_ADDRESS not specified, please correct before proceeding.\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "83340af4",
       "metadata": {
         "id": "83340af4"
       },
@@ -308,73 +356,18 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "4814ea21",
       "metadata": {
         "id": "4814ea21"
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "06571eb4063b"
-      },
-      "source": [
-        "#### UUID\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "4e166d927e36"
-      },
-      "outputs": [],
-      "source": [
-        "import random\n",
-        "import string\n",
-        "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "71404c9f"
-      },
-      "source": [
-        "#### Set your email address\n",
-        "This is used for delivering model monitoring notifications.\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "4b1d2b69"
-      },
-      "outputs": [],
-      "source": [
-        "EMAIL_ADDRESS = \"[your-email-address]\"  # @param {type:\"string\"}\n",
-        "if not EMAIL_ADDRESS or EMAIL_ADDRESS == \"[your-email-address]\":\n",
-        "    print(\"EMAIL_ADDRESS not specified, please correct before proceeding.\")"
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "20a546c3",
       "metadata": {
         "id": "20a546c3"
       },
@@ -382,35 +375,16 @@
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Vertex AI Workbench notebooks**, your environment is already\n",
-        "authenticated.\n",
+        "authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
-        "\n",
-        "**Otherwise**, follow these steps:\n",
-        "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
-        "\n",
-        "2. Click **Create service account**.\n",
-        "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
-        "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
-        "\n",
-        "5. Click **Create**. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
-        "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+        "when prompted to authenticate your account via oAuth.\n"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "06c51076",
       "metadata": {
         "id": "06c51076"
       },
@@ -447,284 +421,73 @@
     },
     {
       "cell_type": "markdown",
+      "id": "6b01af18",
       "metadata": {
-        "id": "bucket:custom"
+        "id": "6b01af18"
       },
       "source": [
-        "### Create a Cloud Storage bucket\n",
+        "### Upload the model\n",
         "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
+        "The churn propensity model you'll be using in this notebook has been trained in BigQuery ML and exported to a Google Cloud Storage bucket. This illustrates how you can easily export a trained model and move a model from one cloud service to another. \n",
         "\n",
-        "Set the name of your Cloud Storage bucket below, which you use in this tutorial to upload the `input schema` for the monitoring service.\n",
-        "\n",
-        "Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "bucket"
-      },
-      "outputs": [],
-      "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "autoset_bucket"
-      },
-      "outputs": [],
-      "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "Next, import the model. **If you've already imported your model, you can skip this step.**"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "9638ad2c",
       "metadata": {
-        "id": "create_bucket"
+        "id": "9638ad2c"
       },
       "source": [
-        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+        "<span id=\"papermill-error-cell\" style=\"color:red; font-family:Helvetica Neue, Helvetica, Arial, sans-serif; font-size:2em;\">Execution using papermill encountered an exception here and stopped:</span>"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "926e3ba8",
       "metadata": {
-        "id": "create_bucket"
+        "id": "926e3ba8"
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "validate_bucket"
-      },
-      "source": [
-        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "validate_bucket"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil ls -al $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "a0d294ff6d10"
-      },
-      "source": [
-        "### Import libraries"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "bd7a633296eb"
-      },
-      "outputs": [],
-      "source": [
-        "import google.cloud.aiplatform as aiplatform\n",
+        "from datetime import datetime\n",
+        "import json\n",
+        "import time\n",
+        "import re\n",
+        "import tensorflow as tf\n",
         "import tensorflow_data_validation as tfdv\n",
-        "from tensorflow_data_validation.utils import io_util\n",
-        "from tensorflow_metadata.proto.v0 import statistics_pb2"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "init_aip:mbsdk,all"
-      },
-      "source": [
-        "### Initialize Vertex AI SDK for Python\n",
-        "\n",
-        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "init_aip:mbsdk,all"
-      },
-      "outputs": [],
-      "source": [
-        "aiplatform.init(project=PROJECT_ID, location=REGION)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "accelerators:training,prediction"
-      },
-      "source": [
-        "#### Set hardware accelerators\n",
-        "\n",
-        "You can set hardware accelerators for prediction (e.g., GPUs) or choose not to use any (CPU). Hardware accelertors lower the latency response for a prediction request. When choosing a hardware accelerators, consider the additional cost trade-off over latency.\n",
-        "\n",
-        "Set the variables `DEPLOY_GPU/DEPLOY_NGPU` to use a container image supporting a GPU and the number of GPUs allocated to the virtual machine (VM) instance. For example, to use a GPU container image with 4 Nvidia Tesla K80 GPUs allocated to each VM, you would specify:\n",
-        "\n",
-        "    (aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80, 4)\n",
-        "\n",
-        "See the [locations where accelerators are available](https://cloud.google.com/vertex-ai/docs/general/locations#accelerators).\n",
-        "\n",
-        "Otherwise specify `(None, None)` to use a container image to run on a CPU."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "xd5PLXDTlugv"
-      },
-      "outputs": [],
-      "source": [
-        "GPU = False\n",
-        "if GPU:\n",
-        "    DEPLOY_GPU, DEPLOY_NGPU = (aiplatform.gapic.AcceleratorType.NVIDIA_TESLA_K80, 1)\n",
-        "else:\n",
-        "    DEPLOY_GPU, DEPLOY_NGPU = (None, None)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "container:training,prediction"
-      },
-      "source": [
-        "#### Set pre-built containers\n",
-        "\n",
-        "Set the pre-built Docker container image for prediction.\n",
-        "\n",
-        "For the latest list, see [Pre-built containers for prediction](https://cloud.google.com/ai-platform-unified/docs/predictions/pre-built-containers)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "1u1mr18jlugv"
-      },
-      "outputs": [],
-      "source": [
-        "if GPU:\n",
-        "    DEPLOY_VERSION = \"tf2-gpu.2-5\"\n",
-        "else:\n",
-        "    DEPLOY_VERSION = \"tf2-cpu.2-5\"\n",
-        "\n",
-        "DEPLOY_IMAGE = \"{}-docker.pkg.dev/vertex-ai/prediction/{}:latest\".format(\n",
-        "    REGION.split(\"-\")[0], DEPLOY_VERSION\n",
-        ")\n",
-        "\n",
-        "print(\"Deployment:\", DEPLOY_IMAGE, DEPLOY_GPU, DEPLOY_NGPU)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "machine:training,prediction"
-      },
-      "source": [
-        "#### Set machine types\n",
-        "\n",
-        "Next, set the machine types to use for training and prediction.\n",
-        "\n",
-        "- Set the variable `DEPLOY_COMPUTE` to configure your compute resources for prediction.\n",
-        " - `machine type`\n",
-        "     - `n1-standard`: 3.75GB of memory per vCPU\n",
-        "     - `n1-highmem`: 6.5GB of memory per vCPU\n",
-        "     - `n1-highcpu`: 0.9 GB of memory per vCPU\n",
-        " - `vCPUs`: number of \\[2, 4, 8, 16, 32, 64, 96 \\]\n",
-        "\n",
-        "*Note: You may also use n2 and e2 machine types for training and deployment, but they do not support GPUs*."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "YAXwbqKKlugv"
-      },
-      "outputs": [],
-      "source": [
-        "MACHINE_TYPE = \"n1-standard\"\n",
-        "\n",
-        "VCPU = \"4\"\n",
-        "TRAIN_COMPUTE = MACHINE_TYPE + \"-\" + VCPU\n",
-        "print(\"Train machine type\", TRAIN_COMPUTE)\n",
-        "\n",
-        "MACHINE_TYPE = \"n1-standard\"\n",
-        "\n",
-        "VCPU = \"4\"\n",
-        "DEPLOY_COMPUTE = MACHINE_TYPE + \"-\" + VCPU\n",
-        "print(\"Deploy machine type\", DEPLOY_COMPUTE)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "9bf06cd476e9"
-      },
-      "source": [
-        "### Upload the model artifacts as a `Vertex AI Model` resource\n",
-        "\n",
-        "First, you upload the pre-trained custom tabular model artifacts as a `Vertex AI Model` resource using the `upload()` method, with the following parameters:\n",
-        "\n",
-        "- `display_name`: The human readable name for the `Model` resource.\n",
-        "- `artifact_uri`: The Cloud Storage location of the model artifacts.\n",
-        "- `serving_container_image`: The serving container image to use when the model is deployed to a `Vertex AI Endpoint` resource.\n",
-        "- `sync`: Whether to wait for the process to complete, or return immediately (async)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "0193f247e216"
-      },
-      "outputs": [],
-      "source": [
-        "MODEL_ARTIFACT_URI = \"gs://mco-mm/churn\"\n",
-        "\n",
-        "model = aiplatform.Model.upload(\n",
-        "    display_name=\"churn_\" + UUID,\n",
-        "    artifact_uri=MODEL_ARTIFACT_URI,\n",
-        "    serving_container_image_uri=DEPLOY_IMAGE,\n",
-        "    sync=True,\n",
-        ")\n",
-        "\n",
-        "print(model)"
+        "from tensorflow_data_validation.utils import io_util \n",
+        "from tensorflow_metadata.proto.v0 import statistics_pb2\n",
+        "\n",
+        "MODEL_DISPLAY_NAME=f\"batch_prediction_monitoring_test_model_{datetime.now().strftime('%Y%m%d%H%M%S')}\"\n",
+        "CONTAINER_IMAGE_URI=\"us-docker.pkg.dev/cloud-aiplatform/prediction/tf2-cpu.2-4:latest\"\n",
+        "ARTIFACT_URI=\"gs://mco-mm/churn\"\n",
+        "\n",
+        "output = ! gcloud ai models upload \\\n",
+        "  --region=$REGION \\\n",
+        "  --display-name=$MODEL_DISPLAY_NAME \\\n",
+        "  --artifact-uri=$ARTIFACT_URI \\\n",
+        "  --container-image-uri=$CONTAINER_IMAGE_URI \\\n",
+        "  --format=\"value(model)\"\n",
+        "MODEL_ID = output[1].split(\"/\")[5]\n",
+        "print(f\"Model {MODEL_ID} created.\")"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "a4305ddf",
       "metadata": {
         "id": "a4305ddf"
       },
       "source": [
-        "## Submit a batch prediction request with model monitoring enabled\n"
+        "## Submit a batch prediction request with model monitoring enabled"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "053fde99",
       "metadata": {
         "id": "053fde99"
       },
@@ -740,6 +503,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "b832ad31",
       "metadata": {
         "id": "b832ad31"
       },
@@ -747,17 +511,20 @@
       "source": [
         "# Copy files to your projects gs bucket to avoid permission issues.\n",
         "# Ignore any error(s) for bucket already exists.\n",
-        "OUTPUT_GS_PATH = f\"{BUCKET_URI}/bp_mm_output\"\n",
-        "INPUT_GS_PATH = f\"{BUCKET_URI}/bp_mm_input\"\n",
+        "OUTPUT_GS_PATH = f\"gs://{PROJECT_ID.replace('-', '_')}_bp_mm_output\"\n",
+        "INPUT_GS_PATH = f\"gs://{PROJECT_ID.replace('-', '_')}_bp_mm_input\"\n",
         "PUBLIC_TRAINING_DATASET = \"gs://bp_mm_public_data/churn/churn_bp_insample.csv\"\n",
         "TRAINING_DATASET = f\"{INPUT_GS_PATH}/churn_bp_insample.csv\"\n",
         "TRAINING_DATASET_FORMAT = \"csv\"\n",
         "\n",
+        "! gsutil mb -p {PROJECT_ID} -l {REGION} -b on {INPUT_GS_PATH}\n",
+        "! gsutil mb -p {PROJECT_ID} -l {REGION} -b on {OUTPUT_GS_PATH}\n",
         "! gsutil copy $PUBLIC_TRAINING_DATASET $INPUT_GS_PATH"
       ]
     },
     {
       "cell_type": "markdown",
+      "id": "34c95126",
       "metadata": {
         "id": "34c95126"
       },
@@ -774,18 +541,21 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "3a54368a",
       "metadata": {
         "id": "3a54368a"
       },
       "outputs": [],
       "source": [
+        "now = datetime.now()\n",
         "INPUT_URI = \"gs://bp_mm_public_data/churn/churn_bp_outsample.jsonl\"\n",
         "OUTPUT_URI = OUTPUT_GS_PATH\n",
         "INSTANCES_FORMAT = \"jsonl\"\n",
         "PREDICTIONS_FORMAT = \"jsonl\"\n",
         "JOB_NAME_PREFIX = \"bp_mm_demo\"\n",
-        "MODEL_NAME = model.resource_name\n",
-        "BATCH_PREDICTION_JOB_NAME = JOB_NAME_PREFIX + \"_\" + UUID\n",
+        "MODEL_NAME = f\"projects/{PROJECT_ID}/locations/{REGION}/models/{MODEL_ID}\"\n",
+        "MACHINE_TYPE = \"n1-standard-8\"\n",
+        "BATCH_PREDICTION_JOB_NAME = JOB_NAME_PREFIX + \"_\" + now.strftime(\"%Y%m%d%H%M%S\")\n",
         "\n",
         "from google.cloud.aiplatform_v1beta1.types import (\n",
         "    BatchDedicatedResources, BatchPredictionJob, GcsDestination, GcsSource,\n",
@@ -803,7 +573,7 @@
         "        gcs_destination=GcsDestination(output_uri_prefix=OUTPUT_URI),\n",
         "    ),\n",
         "    dedicated_resources=BatchDedicatedResources(\n",
-        "        machine_spec=MachineSpec(machine_type=DEPLOY_COMPUTE),\n",
+        "        machine_spec=MachineSpec(machine_type=MACHINE_TYPE),\n",
         "        starting_replica_count=1,\n",
         "        max_replica_count=1,\n",
         "    ),\n",
@@ -834,6 +604,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "cae39778",
       "metadata": {
         "id": "cae39778"
       },
@@ -846,6 +617,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "bcdd4a47",
       "metadata": {
         "id": "bcdd4a47"
       },
@@ -866,6 +638,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "49ec90a0",
       "metadata": {
         "id": "49ec90a0"
       },
@@ -878,6 +651,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "c30496b5",
       "metadata": {
         "id": "c30496b5"
       },
@@ -890,6 +664,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "831651c2",
       "metadata": {
         "id": "831651c2"
       },
@@ -909,6 +684,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "a705c10b",
       "metadata": {
         "id": "a705c10b"
       },
@@ -919,6 +695,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "2bbdddac",
       "metadata": {
         "id": "2bbdddac"
       },
@@ -931,6 +708,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "f6c674e9",
       "metadata": {
         "id": "f6c674e9"
       },
@@ -968,6 +746,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "233b1266",
       "metadata": {
         "id": "233b1266"
       },
@@ -980,6 +759,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "4e8c00a7",
       "metadata": {
         "id": "4e8c00a7"
       },
@@ -994,6 +774,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "497a0016",
       "metadata": {
         "id": "497a0016"
       },
@@ -1009,6 +790,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "eabc3f81",
       "metadata": {
         "id": "eabc3f81"
       },
@@ -1024,6 +806,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "0aa0219d",
       "metadata": {
         "id": "0aa0219d"
       },

notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb

@@ -37,7 +37,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai-samples/main/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",

notebooks/community/prediction/custom_prediction_routines/SDK_Pytorch_Custom_Predict.ipynb

@@ -37,7 +37,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/prediction/custom_prediction_routines/SDK_Pytorch_Custom_Predict.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai-samples/main/notebooks/community/prediction/custom_prediction_routines/SDK_Pytorch_Custom_Predict.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",

notebooks/community/prediction/custom_prediction_routines/SDK_Triton_PyTorch_Local_Prediction.ipynb

@@ -37,7 +37,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/prediction/custom_prediction_routines/SDK_Triton_PyTorch_Local_Prediction.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai-samples/main/notebooks/community/prediction/custom_prediction_routines/SDK_Triton_PyTorch_Local_Prediction.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",

notebooks/community/sdk/SDK_Custom_Container_Prediction.ipynb

@@ -29,23 +29,12 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# Deploying Iris-detection model using FastAPI and Vertex AI custom container serving\n",
-        "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/SDK_Custom_Container_Prediction.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/SDK_Custom_Container_Prediction.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/sdk/SDK_Custom_Container_Prediction.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/SDK_Custom_Container_Prediction.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
         "  </td>\n",
         "</table>"
       ]
@@ -58,67 +47,38 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This tutorial walks you through building a custom container to serve a scikit-learn model on Vertex AI. You use the FastAPI Python web server framework to create a prediction and health endpoint. You also incorporate a pre-processor from training pipeline into your online serving application."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cbd99f7bfc8e"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "The objective of this notebook is to create, deploy and serve a custom classification model on Vertex AI. This notebook focuses more on deploying the model than on the design of the model itself. \n",
+        "This tutorial walks through building a custom container to serve a scikit-learn model on Vertex Predictions. You will use the FastAPI Python web server framework to create a prediction and health endpoint.\n",
+        "You will also cover incorporating a pre-processor from training into your online serving.\n",
         "\n",
         "\n",
-        "This tutorial uses the following Google Cloud ML services and resources:\n",
-        "\n",
-        "- Vertex AI Models\n",
-        "- Vertex AI Endpoints\n",
-        "\n",
-        "The steps performed include:\n",
-        "\n",
-        "- Train a model that uses flower's measurements as input to predict the class of iris.\n",
-        "- Save the model and its serialized pre-processor.\n",
-        "- Build a FastAPI server to handle predictions and health checks.\n",
-        "- Build a custom container with model artifacts.\n",
-        "- Upload and deploy custom container to Vertex AI Endpoints."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "0fe0bb78c9ce"
-      },
-      "source": [
         "### Dataset\n",
         "\n",
-        "This tutorial uses R.A. Fisher's Iris dataset, a small dataset that is a popular choice for trying out machine learning techniques. Each instance has four numerical features, which are different measurements of a flower, and a target label that\n",
+        "This tutorial uses R.A. Fisher's Iris dataset, a small dataset that is popular for trying out machine learning techniques. Each instance has four numerical features, which are different measurements of a flower, and a target label that\n",
         "marks it as one of three types of iris: Iris setosa, Iris versicolour, or Iris virginica.\n",
         "\n",
         "This tutorial uses [the copy of the Iris dataset included in the\n",
-        "scikit-learn library](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_iris.html#sklearn.datasets.load_iris)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "c681f532cf64"
-      },
-      "source": [
+        "scikit-learn library](https://scikit-learn.org/stable/datasets/index.html#iris-dataset).\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "The goal is to:\n",
+        "- Train a model that uses a flower's measurements as input to predict what type of iris it is.\n",
+        "- Save the model and its serialized pre-processor\n",
+        "- Build a FastAPI server to handle predictions and health checks\n",
+        "- Build a custom container with model artifacts\n",
+        "- Upload and deploy custom container to Vertex Prediction\n",
+        "\n",
+        "This tutorial focuses more on deploying this model with Vertex AI than on\n",
+        "the design of the model itself.\n",
+        "\n",
         "### Costs \n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
         "\n",
         "* Vertex AI\n",
-        "* Cloud Storage\n",
-        "* Artifact Registry\n",
-        "* Cloud Build\n",
         "\n",
         "Learn about [Vertex AI\n",
-        "pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage\n",
-        "pricing](https://cloud.google.com/storage/pricing), [Artifact Registry pricing](https://cloud.google.com/artifact-registry/pricing) and [Cloud Build pricing](https://cloud.google.com/build/pricing) and use the [Pricing\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing), and use the [Pricing\n",
         "Calculator](https://cloud.google.com/products/calculator/)\n",
         "to generate a cost estimate based on your projected usage."
       ]
@@ -131,10 +91,8 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "**If you are using Colab or Vertex AI Workbench notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook.\n",
-        "\n",
-        "**If you are using Colab**, docker related steps are skipped as Colab doesn't fully support docker yet."
+        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step."
       ]
     },
     {
@@ -181,7 +139,7 @@
         "id": "i7EUnXsZhAGF"
       },
       "source": [
-        "## Install additional packages\n",
+        "### Install additional packages\n",
         "\n",
         "Install additional package dependencies not installed in your notebook environment, such as NumPy, Scikit-learn, FastAPI, Uvicorn, and joblib. Use the latest major GA version of each package."
       ]
@@ -205,31 +163,18 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "1fd00fa70a2a"
+        "id": "wyy5Lbnzg5fi"
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\"\n",
-        "\n",
         "# Required in Docker serving container\n",
-        "! pip3 install -U {USER_FLAG} -r requirements.txt -q\n",
+        "%pip install -U --user -r requirements.txt\n",
         "\n",
         "# For local FastAPI development and running\n",
-        "! pip3 install -U {USER_FLAG} \"uvicorn[standard]>=0.12.0,<0.14.0\" fastapi~=0.63 -q\n",
+        "%pip install -U --user \"uvicorn[standard]>=0.12.0,<0.14.0\" fastapi~=0.63\n",
         "\n",
         "# Vertex SDK for Python\n",
-        "! pip3 install -U {USER_FLAG} google-cloud-aiplatform -q"
+        "%pip install -U --user google-cloud-aiplatform"
       ]
     },
     {
@@ -285,14 +230,14 @@
         "\n",
         "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "1. [Enable the APIs for Vertex AI, Compute Engine and Artifact Registry](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute.googleapis.com,artifactregistry.googleapis.com).\n",
+        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
         "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
         "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
         "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+        "**Note**: Jupyter runs lines prefixed with `!` or `%` as shell commands, and it interpolates Python variables with `$` or `{}` into these commands."
       ]
     },
     {
@@ -306,17 +251,6 @@
         "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "cde8e0876d62"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -325,11 +259,24 @@
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
-        "    # Get your GCP project id from gcloud\n",
-        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "# Get your Google Cloud project ID from gcloud\n",
+        "shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "\n",
+        "try:\n",
         "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID:\", PROJECT_ID)"
+        "except IndexError:\n",
+        "    PROJECT_ID = None\n",
+        "\n",
+        "print(\"Project ID:\", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qJYoRfYng0XZ"
+      },
+      "source": [
+        "Otherwise, set your project ID here."
       ]
     },
     {
@@ -340,85 +287,28 @@
       },
       "outputs": [],
       "source": [
-        "! gcloud config set project $PROJECT_ID"
+        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
+        "    PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "becb6514d26a"
-      },
-      "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
-        "\n",
-        "- Americas: `us-central1`\n",
-        "- Europe: `europe-west4`\n",
-        "- Asia Pacific: `asia-east1`\n",
-        "\n",
-        "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
-        "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "959545da671a"
-      },
-      "outputs": [],
-      "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "e663bd062c6f"
-      },
-      "source": [
-        "#### UUID\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "953fa6e5ddda"
-      },
-      "outputs": [],
-      "source": [
-        "import random\n",
-        "import string\n",
-        "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "40206eb20b53"
+        "id": "dr--iN2kAylZ"
       },
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench notebooks**, your environment is already\n",
-        "authenticated.\n",
-        "\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sBCra4QMA2wR"
+      },
+      "source": [
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -447,23 +337,20 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "4dd67f16eff2"
+        "id": "PyQmSRbKA8r-"
       },
       "outputs": [],
       "source": [
+        "import os\n",
+        "import sys\n",
+        "\n",
         "# If you are running this notebook in Colab, run this cell and follow the\n",
         "# instructions to authenticate your GCP account. This provides access to your\n",
         "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
         "# requests.\n",
         "\n",
-        "import os\n",
-        "import sys\n",
-        "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -472,10 +359,57 @@
         "    # If you are running this notebook locally, replace the string below with the\n",
         "    # path to your service account key and run this cell to authenticate your GCP\n",
         "    # account.\n",
-        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "    elif not os.getenv(\"IS_TESTING\") and not os.getenv(\n",
+        "        \"GOOGLE_APPLICATION_CREDENTIALS\"\n",
+        "    ):\n",
         "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XoEqT2Y4DJmf"
+      },
+      "source": [
+        "### Configure project and resource names"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MzGDU7TWdts_"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "MODEL_ARTIFACT_DIR = \"custom-container-prediction-model\"  # @param {type:\"string\"}\n",
+        "REPOSITORY = \"custom-container-prediction\"  # @param {type:\"string\"}\n",
+        "IMAGE = \"sklearn-fastapi-server\"  # @param {type:\"string\"}\n",
+        "MODEL_DISPLAY_NAME = \"sklearn-custom-container\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ca1a915d641d"
+      },
+      "source": [
+        "`REGION` - Used for operations\n",
+        "throughout the rest of this notebook. Make sure to [choose a region where Cloud\n",
+        "Vertex AI services are\n",
+        "available](https://cloud.google.com/vertex-ai/docs/general/locations#feature-availability). You may\n",
+        "not use a Multi-Regional Storage bucket for training with Vertex AI.\n",
+        "\n",
+        "`MODEL_ARTIFACT_DIR` - Folder directory path to your model artifacts within a Cloud Storage bucket, for example: \"my-models/fraud-detection/trial-4\"\n",
+        "\n",
+        "`REPOSITORY` - Name of the Artifact Repository to create or use.\n",
+        "\n",
+        "`IMAGE` - Name of the container image that will be pushed.\n",
+        "\n",
+        "`MODEL_DISPLAY_NAME` - Display name of Vertex AI Model resource."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -486,8 +420,8 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "To update your model artifacts without re-building the container, you upload your model\n",
-        "artifacts and any custom code to Cloud Storage bucket.\n",
+        "To update your model artifacts without re-building the container, you must upload your model\n",
+        "artifacts and any custom code to Cloud Storage.\n",
         "\n",
         "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
         "Cloud Storage buckets. "
@@ -501,21 +435,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "db3de5b7b0a4"
-      },
-      "outputs": [],
-      "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -535,7 +455,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -555,95 +475,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d3938f6d37a1"
-      },
-      "source": [
-        "## Import libraries"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "e95ca1e5e07c"
-      },
-      "outputs": [],
-      "source": [
-        "from google.cloud import aiplatform"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "750d53e37094"
-      },
-      "source": [
-        "### Initialize Vertex AI SDK for Python\n",
-        "\n",
-        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "1a3aa2d4a74f"
-      },
-      "outputs": [],
-      "source": [
-        "aiplatform.init(project=PROJECT_ID, location=REGION)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "XoEqT2Y4DJmf"
-      },
-      "source": [
-        "### Configure resource names\n",
-        "\n",
-        "Set a name for the following resources:\n",
-        "\n",
-        "`MODEL_ARTIFACT_DIR` - Folder directory path to your model artifacts within a Cloud Storage bucket, for example: \"my-models/fraud-detection/trial-4\"\n",
-        "\n",
-        "`REPOSITORY` - Name of the Artifact Repository to create or use.\n",
-        "\n",
-        "`IMAGE` - Name of the container image that is pushed to the repository.\n",
-        "\n",
-        "`MODEL_DISPLAY_NAME` - Display name of Vertex AI Model resource."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "MzGDU7TWdts_"
-      },
-      "outputs": [],
-      "source": [
-        "MODEL_ARTIFACT_DIR = \"[your-artifact-directory]\"  # @param {type:\"string\"}\n",
-        "REPOSITORY = \"[your-repository-name]\"  # @param {type:\"string\"}\n",
-        "IMAGE = \"[your-image-name]\"  # @param {type:\"string\"}\n",
-        "MODEL_DISPLAY_NAME = \"[your-model-display-name]\"  # @param {type:\"string\"}\n",
-        "\n",
-        "# Set the defaults if no names were specified\n",
-        "if MODEL_ARTIFACT_DIR == \"[your-artifact-directory]\":\n",
-        "    MODEL_ARTIFACT_DIR = \"custom-container-prediction-model\"\n",
-        "\n",
-        "if REPOSITORY == \"[your-repository-name]\":\n",
-        "    REPOSITORY = \"custom-container-prediction\"\n",
-        "\n",
-        "if IMAGE == \"[your-image-name]\":\n",
-        "    IMAGE = \"sklearn-fastapi-server\"\n",
-        "\n",
-        "if MODEL_DISPLAY_NAME == \"[your-model-display-name]\":\n",
-        "    MODEL_DISPLAY_NAME = \"sklearn-custom-container\""
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -653,9 +485,9 @@
       },
       "source": [
         "## Write your pre-processor\n",
-        "Standardize the training data so each numerical feature column has a mean of 0 and a standard deviation of 1 [can improve your model](https://developers.google.com/machine-learning/crash-course/representation/cleaning-data).\n",
+        "Scaling training data so each numerical feature column has a mean of 0 and a standard deviation of 1 [can improve your model](https://developers.google.com/machine-learning/crash-course/representation/cleaning-data).\n",
         "\n",
-        "Define a `app` folder and create `preprocess.py`, which contains a class to perform standardization."
+        "Create `preprocess.py`, which contains a class to do this scaling:"
       ]
     },
     {
@@ -706,7 +538,7 @@
         "## Train and store model with pre-processor\n",
         "Next, use `preprocess.MySimpleScaler` to preprocess the iris data, then train a model using scikit-learn.\n",
         "\n",
-        "At the end, export your trained model as a joblib (`.joblib`) file and export your `MySimpleScaler` instance as a pickle (`.pkl`) file."
+        "At the end, export your trained model as a joblib (`.joblib`) file and export your `MySimpleScaler` instance as a pickle (`.pkl`) file:"
       ]
     },
     {
@@ -764,7 +596,7 @@
       },
       "outputs": [],
       "source": [
-        "!gsutil cp model.joblib preprocessor.pkl {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n",
+        "!gsutil cp model.joblib preprocessor.pkl {BUCKET_NAME}/{MODEL_ARTIFACT_DIR}/\n",
         "%cd .."
       ]
     },
@@ -774,9 +606,7 @@
         "id": "480a1d88ecdb"
       },
       "source": [
-        "## Build a FastAPI server\n",
-        "\n",
-        "To serve predictions from the classification model, build a FastAPI server application."
+        "## Build a FastAPI server"
       ]
     },
     {
@@ -844,7 +674,7 @@
       },
       "source": [
         "### Add pre-start script\n",
-        "FastAPI executes the following script before starting up the server. The `PORT` environment variable is set to equal to `AIP_HTTP_PORT` in order to run FastAPI on the same port expected by Vertex AI."
+        "FastAPI will execute this script before starting up the server. The `PORT` environment variable is set to equal `AIP_HTTP_PORT` in order to run FastAPI on same the port expected by Vertex AI."
       ]
     },
     {
@@ -866,7 +696,7 @@
         "id": "8b62ddf1def3"
       },
       "source": [
-        "### Create test instances\n",
+        "### Store test instances to use later\n",
         "To learn more about formatting input instances in JSON, [read the documentation.](https://cloud.google.com/vertex-ai/docs/predictions/online-predictions-custom-models#request-body-details)"
       ]
     },
@@ -896,13 +726,35 @@
         "## Build and push container to Artifact Registry"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3bdb9a7768a5"
+      },
+      "source": [
+        "### Build your container\n",
+        "Optionally copy in your credentials to run the container locally."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fbb77f4f56c7"
+      },
+      "outputs": [],
+      "source": [
+        "# NOTE: Copy in credentials to run locally, this step can be skipped for deployment\n",
+        "%cp $GOOGLE_APPLICATION_CREDENTIALS app/credentials.json"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "240578ec9efe"
       },
       "source": [
-        "Write the `Dockerfile`, using `tiangolo/uvicorn-gunicorn-fastapi` as a base image. This automatically runs FastAPI for you using Gunicorn and Uvicorn. Visit [the FastAPI docs to read more about deploying FastAPI with Docker](https://fastapi.tiangolo.com/deployment/docker/) to learn more."
+        "Write the Dockerfile, using `tiangolo/uvicorn-gunicorn-fastapi` as a base image. This will automatically run FastAPI for you using Gunicorn and Uvicorn. Visit [the FastAPI docs to read more about deploying FastAPI with Docker](https://fastapi.tiangolo.com/deployment/docker/)."
       ]
     },
     {
@@ -915,7 +767,7 @@
       "source": [
         "%%writefile Dockerfile\n",
         "\n",
-        "FROM tiangolo/uvicorn-gunicorn-fastapi:python3.9\n",
+        "FROM tiangolo/uvicorn-gunicorn-fastapi:python3.7\n",
         "\n",
         "COPY ./app /app\n",
         "COPY requirements.txt requirements.txt\n",
@@ -929,11 +781,7 @@
         "id": "04c988201499"
       },
       "source": [
-        "### Build the image locally (optional)\n",
-        "\n",
-        "Build the image using docker to test it locally.\n",
-        "\n",
-        "**Note:** Docker is only being used to test the container locally. For deployment to Artifact registry, Cloud-Build is used."
+        "Build the image and tag the Artifact Registry path that you will push to."
       ]
     },
     {
@@ -944,10 +792,9 @@
       },
       "outputs": [],
       "source": [
-        "if not IS_COLAB and not os.getenv(\"IS_TESTING\"):\n",
-        "    ! sudo docker build \\\n",
-        "            --tag=\"{REGION}-docker.pkg.dev/{PROJECT_ID}/{REPOSITORY}/{IMAGE}\" \\\n",
-        "            ."
+        "!docker build \\\n",
+        "    --tag={REGION}-docker.pkg.dev/{PROJECT_ID}/{REPOSITORY}/{IMAGE} \\\n",
+        "    ."
       ]
     },
     {
@@ -958,7 +805,7 @@
       "source": [
         "### Run and test the container locally (optional)\n",
         "\n",
-        "Test running the container locally in detached mode and provide the environment variables that the container requires. These variables are provided to the container by Vertex AI once deployed. Test the `/health` and `/predict` routes and then stop the running image."
+        "Run the container locally in detached mode and provide the environment variables that the container requires. These env vars will be provided to the container by Vertex Prediction once deployed. Test the `/health` and `/predict` routes, then stop the running image."
       ]
     },
     {
@@ -969,25 +816,15 @@
       },
       "outputs": [],
       "source": [
-        "if not IS_COLAB and not os.getenv(\"IS_TESTING\"):\n",
-        "    ! sudo docker stop local-iris\n",
-        "    ! sudo docker rm local-iris\n",
-        "    ! sudo docker run -d -p 80:8080 \\\n",
-        "            --name=local-iris \\\n",
-        "            -e AIP_HTTP_PORT=8080 \\\n",
-        "            -e AIP_HEALTH_ROUTE=/health \\\n",
-        "            -e AIP_PREDICT_ROUTE=/predict \\\n",
-        "            -e AIP_STORAGE_URI={BUCKET_URI}/{MODEL_ARTIFACT_DIR} \\\n",
-        "            \"{REGION}-docker.pkg.dev/{PROJECT_ID}/{REPOSITORY}/{IMAGE}\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "248f481e8e90"
-      },
-      "source": [
-        "Ping the health route."
+        "!docker rm local-iris\n",
+        "!docker run -d -p 80:8080 \\\n",
+        "    --name=local-iris \\\n",
+        "    -e AIP_HTTP_PORT=8080 \\\n",
+        "    -e AIP_HEALTH_ROUTE=/health \\\n",
+        "    -e AIP_PREDICT_ROUTE=/predict \\\n",
+        "    -e AIP_STORAGE_URI={BUCKET_NAME}/{MODEL_ARTIFACT_DIR} \\\n",
+        "    -e GOOGLE_APPLICATION_CREDENTIALS=credentials.json \\\n",
+        "    {REGION}-docker.pkg.dev/{PROJECT_ID}/{REPOSITORY}/{IMAGE}"
       ]
     },
     {
@@ -998,17 +835,7 @@
       },
       "outputs": [],
       "source": [
-        "if not IS_COLAB and not os.getenv(\"IS_TESTING\"):\n",
-        "    ! curl localhost/health"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "2d6821fb2b7d"
-      },
-      "source": [
-        "Pass the `instances.json` and test the predict route."
+        "!curl localhost/health"
       ]
     },
     {
@@ -1019,20 +846,10 @@
       },
       "outputs": [],
       "source": [
-        "if not IS_COLAB and not os.getenv(\"IS_TESTING\"):\n",
-        "    ! curl -X POST \\\n",
-        "      -d @instances.json \\\n",
-        "      -H \"Content-Type: application/json; charset=utf-8\" \\\n",
-        "      localhost/predict"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d1d6ee697180"
-      },
-      "source": [
-        "Stop and delete the container locally."
+        "!curl -X POST \\\n",
+        "  -d @instances.json \\\n",
+        "  -H \"Content-Type: application/json; charset=utf-8\" \\\n",
+        "  localhost/predict"
       ]
     },
     {
@@ -1043,9 +860,7 @@
       },
       "outputs": [],
       "source": [
-        "if not IS_COLAB and not os.getenv(\"IS_TESTING\"):\n",
-        "    ! sudo docker stop local-iris\n",
-        "    ! sudo docker rm local-iris"
+        "!docker stop local-iris"
       ]
     },
     {
@@ -1056,32 +871,31 @@
       "source": [
         "### Push the container to artifact registry\n",
         "\n",
-        "Create your repository in the Artifact registry and push your container image to the repository.\n",
-        "Run this below cell once to create the artifact repository."
+        "Configure Docker to access Artifact Registry. Then push your container image to your Artifact Registry repository."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "5f98a42332e5"
+        "id": "09ffe2434e3d"
       },
       "outputs": [],
       "source": [
-        "!gcloud artifacts repositories create {REPOSITORY} \\\n",
+        "!gcloud beta artifacts repositories create {REPOSITORY} \\\n",
         "    --repository-format=docker \\\n",
         "    --location=$REGION"
       ]
     },
     {
-      "cell_type": "markdown",
+      "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "id": "d2e0b8b700aa"
+        "id": "293437024749"
       },
+      "outputs": [],
       "source": [
-        "Push the image to the created artifact repository using Cloud-Build.\n",
-        "\n",
-        "**Note:** The following command automatically considers the Dockerfile from the directory it is being run from."
+        "!gcloud auth configure-docker {REGION}-docker.pkg.dev"
       ]
     },
     {
@@ -1092,7 +906,7 @@
       },
       "outputs": [],
       "source": [
-        "!gcloud builds submit --region={REGION} --tag={REGION}-docker.pkg.dev/{PROJECT_ID}/{REPOSITORY}/{IMAGE}"
+        "!docker push {REGION}-docker.pkg.dev/{PROJECT_ID}/{REPOSITORY}/{IMAGE}"
       ]
     },
     {
@@ -1101,7 +915,9 @@
         "id": "b438bfa2129f"
       },
       "source": [
-        "## Deploy to Vertex AI"
+        "## Deploy to Vertex AI\n",
+        "\n",
+        "Use the Python SDK to upload and deploy your model."
       ]
     },
     {
@@ -1110,8 +926,29 @@
         "id": "4ae19df6a33e"
       },
       "source": [
-        "### Create Vertex AI model using artifact uri\n",
-        "Use the Python SDK to upload and deploy your model from the artifact registry."
+        "### Upload the custom container model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8d682d8388ec"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "574fb82d3eed"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT, location=REGION)"
       ]
     },
     {
@@ -1124,7 +961,7 @@
       "source": [
         "model = aiplatform.Model.upload(\n",
         "    display_name=MODEL_DISPLAY_NAME,\n",
-        "    artifact_uri=f\"{BUCKET_URI}/{MODEL_ARTIFACT_DIR}\",\n",
+        "    artifact_uri=f\"{BUCKET_NAME}/{MODEL_ARTIFACT_DIR}\",\n",
         "    serving_container_image_uri=f\"{REGION}-docker.pkg.dev/{PROJECT_ID}/{REPOSITORY}/{IMAGE}\",\n",
         ")"
       ]
@@ -1135,9 +972,8 @@
         "id": "bd1b85afc7df"
       },
       "source": [
-        "### Deploy the model to Vertex AI Endpoints\n",
-        "\n",
-        "Deploy the model to a Vertex AI Endpoint. After this step completes, the model is deployed and ready for online predictions."
+        "### Deploy the model on Vertex AI\n",
+        "After this step completes, the model is deployed and ready for online prediction."
       ]
     },
     {
@@ -1157,13 +993,9 @@
         "id": "6883e7b07143"
       },
       "source": [
-        "## Request predictions\n",
+        "## Send predictions\n",
         "\n",
-        "Send online requests to the model deployed to the endpoint and get predictions.\n",
-        "\n",
-        "### Using Python SDK\n",
-        "\n",
-        "Get predictions from the endpoint for a sample input using python SDK."
+        "### Using Python SDK"
       ]
     },
     {
@@ -1183,9 +1015,7 @@
         "id": "370d22f53427"
       },
       "source": [
-        "### Using REST\n",
-        "\n",
-        "Get predictions from the endpoint using curl request."
+        "### Using REST"
       ]
     },
     {
@@ -1220,9 +1050,7 @@
         "id": "fa71174a7dd0"
       },
       "source": [
-        "### Using gcloud CLI\n",
-        "\n",
-        "Get predictions from the endpoint using gcloud CLI."
+        "### Using gcloud CLI"
       ]
     },
     {
@@ -1233,7 +1061,7 @@
       },
       "outputs": [],
       "source": [
-        "!gcloud ai endpoints predict $ENDPOINT_ID \\\n",
+        "!gcloud beta ai endpoints predict $ENDPOINT_ID \\\n",
         "  --region=$REGION \\\n",
         "  --json-request=instances.json"
       ]
@@ -1249,13 +1077,7 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
-        "\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- Artifact Registry Image\n",
-        "- Artifact Repository: Set `delete_art_repo` to **True** to delete the repository created in this tutorial.\n",
-        "- Cloud Storage bucket: Set `delete_bucket` to **True** to delete the Cloud Storage bucket used in this tutorial."
+        "Otherwise, you can delete the individual resources you created in this tutorial:"
       ]
     },
     {
@@ -1266,36 +1088,24 @@
       },
       "outputs": [],
       "source": [
-        "delete_bucket = False\n",
-        "delete_art_repo = False\n",
-        "    \n",
         "# Undeploy model and delete endpoint\n",
-        "endpoint.undeploy_all()\n",
-        "endpoint.delete()\n",
+        "endpoint.delete(force=True)\n",
         "\n",
-        "#Delete the model resource\n",
+        "# Delete the model resource\n",
         "model.delete()\n",
         "\n",
         "# Delete the container image from Artifact Registry\n",
         "!gcloud artifacts docker images delete \\\n",
         "    --quiet \\\n",
         "    --delete-tags \\\n",
-        "    {REGION}-docker.pkg.dev/{PROJECT_ID}/{REPOSITORY}/{IMAGE}\n",
-        "\n",
-        "# Delete the artifact registry\n",
-        "if delete_art_repo or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gcloud artifacts repositories delete {REPOSITORY} --location=$REGION -q\n",
-        "    \n",
-        "# Delete the Cloud Storage bucket\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil -m rm -r $BUCKET_URI"
+        "    {REGION}-docker.pkg.dev/{PROJECT_ID}/{REPOSITORY}/{IMAGE}"
       ]
     }
   ],
   "metadata": {
     "colab": {
       "collapsed_sections": [],
-      "name": "SDK_Custom_Container_Prediction.ipynb",
+      "name": "AI_Platform_(Unified)_SDK_Custom_Container_Prediction.ipynb",
       "toc_visible": true
     },
     "kernelspec": {

notebooks/curated.csv

@@ -1,34 +0,0 @@
-tag,notebook,doc
-"AutoML, Text data",official/automl/automl-text-classification.ipynb,vertex-ai/docs/text-data/classification/train-model
-"AutoML, Tabular data",official/automl/sdk_automl_tabular_forecasting_batch.ipynb,vertex-ai/docs/tabular-data/forecasting/tutorials-samples
-"BigQuery, Vertex AI Workbench",official/workbench/exploratory_data_analysis/explore_data_in_bigquery_with_workbench.ipynb,
-"BigQuery ML, Vertex AI Model Registry, Batch prediction",official/model-registry/bqml-vertexai-model-registry.ipynb,
-"BigQuery ML, Vertex AI Model Registry, Online prediction",official/bigquery_ml/bqml-online-prediction.ipynb,
-Custom Training,official/custom/sdk-custom-image-classification-batch.ipynb,
-Custom Training,official/custom/sdk-custom-image-classification-online.ipynb,
-Tabular Data,official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb,vertex-ai/docs/tabular-data/forecasting-arima/overview
-"AutoML, Tabular Data",official/automl/automl_tabular_on_vertex_pipelines.ipynb,vertex-ai/docs/tabular-data/tabular-workflows/e2e-automl
-Vertex AI Experiments,official/experiments/comparing_pipeline_runs.ipynb,
-Vertex AI Experiments,official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb,
-Vertex AI Experiments,official/experiments/comparing_local_trained_models.ipynb,
-Vertex AI Feature Store,official/feature_store/sdk-feature-store.ipynb,
-Matching Engine,official/matching_engine/sdk_matching_engine_for_indexing.ipynb,
-Model Monitoring,official/model_monitoring/model_monitoring.ipynb,
-Vertex AI Pipelines,official/pipelines/lightweight_functions_component_io_kfp.ipynb,
-"Vertex AI Pipelines Image data",official/pipelines/google_cloud_pipeline_components_automl_images.ipynb,
-"Vertex AI Pipelines, Tabular data",official/pipelines/automl_tabular_classification_beans.ipynb,
-"Vertex AI Pipelines, Tabular data",official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb,
-"Vertex AI Pipelines, Text data",official/pipelines/google_cloud_pipeline_components_automl_text.ipynb,
-Vertex AI Pipelines,official/pipelines/custom_model_training_and_batch_prediction.ipynb,
-Vertex AI Pipelines,official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb,
-Vertex AI Pipelines,official/pipelines/control_flow_kfp.ipynb,
-Vertex AI Pipelines,official/pipelines/metrics_viz_run_compare_kfp.ipynb,
-Vertex AI Pipelines,official/pipelines/pipelines_intro_kfp.ipynb,
-Vertex AI Vizier,official/vizier/gapic-vizier-multi-objective-optimization.ipynb,vertex-ai/docs/vizier/using-vizier
-"Vertex Explainable AI, Tabular data",official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb,vertex-ai/docs/explainable-ai/overview
-"Vertex Explainable AI, Tabular data",official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb,vertex-ai/docs/explainable-ai/overview
-"Vertex Explainable AI, Image data",official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb,vertex-ai/docs/explainable-ai/overview
-"Vertex Explainable AI, Image data",official/explainable_ai/sdk_custom_image_classification_online_explain.ipynb,vertex-ai/docs/explainable-ai/overview
-"Vertex Explainable AI, Tabular data",official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb,vertex-ai/docs/explainable-ai/overview
-"Vertex Explainable AI, Tabular data",official/explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb,vertex-ai/docs/explainable-ai/overview
-Vertex ML Metadata,official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb,

notebooks/notebook_template.ipynb

@@ -29,8 +29,6 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# [TODO] Add your H1 title heading here\n",
-        "\n",
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
@@ -82,7 +80,7 @@
         "\n",
         "The steps performed include:\n",
         "\n",
-        "- *{TODO: Add high level bullets for the steps of performed in the notebook}*"
+        "- * {TODO: Add high level bullets for the steps of performed in the notebook}"
       ]
     },
     {
@@ -111,24 +109,21 @@
         "This tutorial uses billable components of Google Cloud:\n",
         "\n",
         "* Vertex AI\n",
-        "* {TODO: BigQyuery}\n",
         "* Cloud Storage\n",
         "\n",
-        "{TODO: Include links to pricing documentation for each product you listed above.\n",
-        " NOTE: If you use BigQuery or Dataflow, you need to add this to the pricing.\n",
-        "}\n",
+        "{TODO: Include links to pricing documentation for each product you listed above.}\n",
         "\n",
-        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing),\n",
-        "{ TODO: [BigQuery pricing](https://cloud.google.com/bigquery/pricing), }\n",
-        "and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), \n",
-        "and use the [Pricing Calculator](https://cloud.google.com/products/calculator/)\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
         "to generate a cost estimate based on your projected usage."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "gCuSR8GkAgzl"
+        "id": "ze4-nDLfK4pw"
       },
       "source": [
         "### Set up your local development environment\n",
@@ -137,17 +132,6 @@
         "all the requirements to run this notebook. You can skip this step."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "24743cf4a1e1"
-      },
-      "source": [
-        "**_NOTE_**: This notebook has been tested in the following environment:\n",
-        "\n",
-        "* Python version = 3.9"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -158,6 +142,10 @@
         "You need the following:\n",
         "\n",
         "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
         "\n",
         "The Google Cloud guide to [Setting up a Python development\n",
         "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
@@ -216,7 +204,7 @@
         "    USER_FLAG = \"--user\"\n",
         "\n",
         "! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
-        "# TODO: Add remaining package installs here. All packages should be on a single pip install to resolve dependencies"
+        "# TODO: Add remaining package installs here"
       ]
     },
     {
@@ -249,14 +237,21 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lWEdiXsJg0XY"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "BF1j6f9HApxa"
       },
       "source": [
-        "## Before you begin\n",
-        "\n",
         "### Set up your Google Cloud project\n",
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
@@ -417,14 +412,21 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "sBCra4QMA2wR"
+        "id": "dr--iN2kAylZ"
       },
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. \n",
-        "\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sBCra4QMA2wR"
+      },
+      "source": [
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -479,7 +481,7 @@
         "    # path to your service account key and run this cell to authenticate your GCP\n",
         "    # account.\n",
         "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS '[your-service-account-key-path]'"
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
       ]
     },
     {
@@ -495,7 +497,7 @@
         "\n",
         "{TODO: Adjust wording in the first paragraph to fit your use case - explain how your tutorial uses the Cloud Storage bucket. The example below shows how Vertex AI uses the bucket for training.}\n",
         "\n",
-        "When you submit a training job using the Vertex AI SDK, you upload a Python package\n",
+        "When you submit a training job using the Cloud SDK, you upload a Python package\n",
         "containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
         "the code from this package. In this tutorial, Vertex AI also saves the\n",
         "trained model that results from your job in the same bucket. Using this model artifact, you can then\n",

notebooks/notebook_template_review.py

@@ -3,33 +3,24 @@ import argparse
 import json
 import os
 import urllib.request
-import csv
 
 parser = argparse.ArgumentParser()
 parser.add_argument('--notebook-dir', dest='notebook_dir',
                     default=None, type=str, help='Notebook directory')
 parser.add_argument('--notebook', dest='notebook',
                     default=None, type=str, help='Notebook to review')
-parser.add_argument('--notebook-file', dest='notebook_file',
-                    default=None, type=str, help='File with list of notebooks to review')
-parser.add_argument('--errors', dest='errors', action='store_true', 
-                    default=False, help='Report errors')
-parser.add_argument('--errors-csv', dest='errors_csv', action='store_true', 
-                    default=False, help='Report errors as CSV')
+parser.add_argument('--errors', dest='errors',
+                    default=False, type=bool, help='Report errors')
+parser.add_argument('--errors-csv', dest='errors_csv',
+                    default=False, type=bool, help='Report errors as CSV')
 parser.add_argument('--errors-codes', dest='errors_codes',
                     default=None, type=str, help='Report only specified errors')
-parser.add_argument('--title', dest='title', action='store_true',
-                    default=False, help='Output description')
-parser.add_argument('--desc', dest='desc', action='store_true', 
-                    default=False, help='Output description')
-parser.add_argument('--uses', dest='uses', action='store_true', 
-                    default=False, help='Output uses (resources)')
-parser.add_argument('--steps', dest='steps', action='store_true', 
-                    default=False, help='Ouput steps')
-parser.add_argument('--web', dest='web', action='store_true', 
-                    default=False, help='Output format in HTML')
-parser.add_argument('--repo', dest='repo', action='store_true', 
-                    default=False, help='Output format in Markdown')
+parser.add_argument('--desc', dest='desc',
+                    default=False, type=bool, help='Output description')
+parser.add_argument('--uses', dest='uses',
+                    default=False, type=bool, help='Output uses (resources)')
+parser.add_argument('--steps', dest='steps',
+                    default=False, type=bool, help='Ouput steps')
 args = parser.parse_args()
 
 if args.errors_codes:
@@ -39,20 +30,6 @@ if args.errors_codes:
 if args.errors_csv:
     args.errors = True
 
-# Copyright cell
-ERROR_COPYRIGHT = 0
-
-# Links cell
-ERROR_TITLE_HEADING = 1
-ERROR_HEADING_CASE = 2
-ERROR_HEADING_CAP = 3
-ERROR_LINK_GIT_MISSING = 4
-ERROR_LINK_COLAB_MISSING = 5
-ERROR_LINK_WORKBENCH_MISSING = 6
-ERROR_LINK_GIT_BAD = 7
-ERROR_LINK_COLAB_BAD = 8
-ERROR_LINK_WORKBENCH_BAD = 9
-
 def parse_dir(directory):
     entries = os.scandir(directory)
     for entry in entries:
@@ -79,8 +56,8 @@ def parse_notebook(path):
         # cell 1 is copyright
         nth = 0
         cell, nth = get_cell(path, cells, nth)
-        if not 'Copyright' in cell['source'][0]:
-            report_error(path, ERROR_COPYRIGHT, "missing copyright cell")
+        if not cell['source'][0].startswith('# Copyright'):
+            report_error(path, 0, "missing copyright cell")
             
         # check for notices
         cell, nth = get_cell(path, cells, nth)
@@ -89,59 +66,40 @@ def parse_notebook(path):
             
         # cell 2 is title and links
         if not cell['source'][0].startswith('# '):
-            report_error(path, ERROR_TITLE_HEADING, "title cell must start with H1 heading")
-            title = ''
+            report_error(path, 1, "title cell must start with H1 heading")
         else:
             title = cell['source'][0][2:].strip()
             check_sentence_case(path, title)
-            
-            # H1 title only
-            if len(cell['source']) == 1:
-                cell, nth = get_cell(path, cells, nth)
            
         # check links.
         source = ''
-        git_link = None
-        colab_link = None
-        workbench_link = None
         for line in cell['source']:
             source += line
             if '<a href="https://github.com' in line:
-                git_link = line.strip()[9:-2].replace('" target="_blank', '')
+                link = line.strip()[9:-2]
                 try:
-                    code = urllib.request.urlopen(git_link).getcode()
+                    code = urllib.request.urlopen(link).getcode()
                 except Exception as e:
-                    # if new notebook
-                    derived_link = os.path.join('https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/', path)
-                    if git_link != derived_link:
-                        report_error(path, ERROR_LINK_GIT_BAD, f"bad GitHub link: {git_link}")
-                    
+                    report_error(path, 7, f"bad GitHub link: {link}")
             if '<a href="https://colab.research.google.com/' in line:
-                colab_link = 'https://github.com/' + line.strip()[50:-2].replace('" target="_blank', '')
+                link = 'https://github.com/' + line.strip()[50:-2]
                 try:
-                    code = urllib.request.urlopen(colab_link).getcode()
+                    code = urllib.request.urlopen(link).getcode()
                 except Exception as e:
-                    # if new notebook
-                    derived_link = os.path.join('https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks', path)
-                    if colab_link != derived_link:
-                        report_error(path, ERROR_LINK_COLAB_BAD, f"bad Colab link: {colab_link}")
-                    
-
+                    report_error(path, 8, f"bad Colab link: {link}")
             if '<a href="https://console.cloud.google.com/vertex-ai/workbench/' in line:
-                workbench_link = line.strip()[91:-2].replace('" target="_blank', '')
+                link = line.strip()[91:-2]
                 try:
-                    code = urllib.request.urlopen(workbench_link).getcode()
+                    code = urllib.request.urlopen(link).getcode()
                 except Exception as e:
-                    derived_link = os.path.join('https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/', path)
-                    if colab_link != workbench_link:
-                        report_error(path, ERROR_LINK_WORKBENCH_BAD, f"bad Workbench link: {workbench_link}")
+                    report_error(path, 9, f"bad Workbench link: {link}")
 
-        if 'View on GitHub' not in source or not git_link:
-            report_error(path, ERROR_LINK_GIT_MISSING, 'Missing link for GitHub')
-        if 'Run in Colab' not in source or not colab_link:
-            report_error(path, ERROR_LINK_COLAB_MISSING, 'Missing link for Colab')    
-        if 'Open in Vertex AI Workbench' not in source or not workbench_link:
-            report_error(path, ERROR_LINK_WORKBENCH_MISSING, 'Missing link for Workbench')
+        if 'View on GitHub' not in source:
+            report_error(path, 4, 'Missing link for GitHub')
+        if 'Open in Vertex AI Workbench' not in source:
+            report_error(path, 5, 'Missing link for Workbench')
+        if 'master' in source:
+            report_error(path, 6, 'Outdated branch (master) used in link')
             
         # Overview
         cell, nth = get_cell(path, cells, nth)
@@ -155,7 +113,7 @@ def parse_notebook(path):
             costs = []
         else:
             desc, uses, steps, costs = parse_objective(path, cell)
-            add_index(path, tag, title, desc, uses, steps, git_link, colab_link, workbench_link)
+            add_index(path, title, desc, uses, steps)
             
         # (optional) Recommendation
         cell, nth = get_cell(path, cells, nth)
@@ -329,10 +287,7 @@ def check_text_cell(path, cell):
         'Vertex Private Endpoint': 'Vertex AI Private Endpoint',
         'Tensorflow': 'TensorFlow',
         'Tensorboard': 'TensorBoard',
-        'Google Cloud Notebooks': 'Vertex AI Workbench Notebooks',
-        'Bigquery': 'BigQuery',
-        'Pytorch': 'PyTorch',
-        'Sklearn': 'scikit-learn'
+        'Google Cloud Notebooks': 'Vertex AI Workbench Notebooks'
     }
     
     for line in cell['source']:
@@ -351,15 +306,15 @@ def check_text_cell(path, cell):
 def check_sentence_case(path, heading):
     words = heading.split(' ')
     if not words[0][0].isupper():
-        report_error(path, ERROR_HEADING_CAP, f"heading must start with capitalized word: {words[0]}")
+        report_error(path, 2, f"heading must start with capitalized word: {words[0]}")
         
     for word in words[1:]:
         word = word.replace(':', '').replace('(', '').replace(')', '')
         if word in ['E2E', 'Vertex', 'AutoML', 'ML', 'AI', 'GCP', 'API', 'R', 'CMEK', 'TFX', 'TFDV', 'SDK',
-                    'VM', 'CPR', 'NVIDIA', 'ID', 'DASK', 'ARIMA_PLUS', 'KFP', 'I/O']:
+                    'VM', 'CPR', 'NVIDIA', 'ID', 'DASK']:
             continue
         if word.isupper():
-            report_error(path, ERROR_HEADING_CASE, f"heading is not sentence case: {word}")
+            report_error(path, 3, f"heading is not sentence case: {word}")
 
 
 def report_error(notebook, code, msg):
@@ -397,9 +352,6 @@ def parse_objective(path, cell):
             continue
             
         if in_desc:
-            if len(desc) > 0 and line.strip() == '':
-                in_desc = False
-                continue
             desc += line
         elif in_uses:
             sline = line.strip()
@@ -420,13 +372,6 @@ def parse_objective(path, cell):
             
     if desc == '':
         report_error(path, 17, "Objective section missing desc")
-    else:
-        desc = desc.lstrip()
-        sentences = desc.split('.')
-        if len(sentences) > 1:
-            desc = sentences[0] + '.\n'
-        if desc.startswith('In this tutorial, you learn') or desc.startswith('In this notebook, you learn'):
-            desc = desc[22].upper() + desc[23:]
         
     if uses == '':
         report_error(path, 18, "Objective section missing uses services list")
@@ -443,102 +388,35 @@ def parse_objective(path, cell):
             
     return desc, uses, steps, costs
 
-def add_index(path, tag, title, desc, uses, steps, git_link, colab_link, workbench_link):
+def add_index(path, title, desc, uses, steps):
+    if not args.desc and not args.uses and not args.steps:
+        return
+    
     title = title.split(':')[-1].strip()
     title = title[0].upper() + title[1:]
-    if args.web:
-        title = title.replace('`', '')
+    
+    print(f"\n[{title}]({path})\n")
+    
+    if args.desc:
+        print(desc)
         
-        print('    <tr>')
-        print('        <td>')
-        tags = tag.split(',')
-        for tag in tags:
-            print(f'            {tag.strip()}<br/>\n')
-        print('        </td>')
-        print('        <td>')
-        print(f'            {title}<br/>\n')
-        if args.desc:
-            desc = desc.replace('`', '')
-            print(f'            {desc}<br/>\n')
-        if linkback:
-            text = ''
-            for tag in tags:
-                text += tag.strip() + ' '
-                
-            print(f'            Learn more about <a src="https://cloud.google.com/{linkback}">{text}</a><br/>\n')
-        print('        </td>')
-        print('        <td>')
-        if colab_link:
-            print(f'            <a src="{colab_link}">Colab</a><br/>\n')
-        if git_link:
-            print(f'            <a src="{git_link}">GitHub</a><br/>\n')
-        if workbench_link:
-            print(f'            <a src="{workbench_link}">Vertex AI Workbench</a><br/>\n')
-        print('        </td>')
-        print('    </tr>\n')
-    elif args.repo:
-        tags = tag.split(',')
-        try:
-            last_tag
-        except:
-            last_tag = ''
-        if tags != last_tag:
-            last_tag = tags
-            flat_list = ''
-            for item in tags:
-                flat_list += item.replace("'", '') + ' '
-            print(f"\n### {flat_list}\n")
-        print(f"\n[{title}]({path})\n")
-    
-        if args.desc:
-            print(desc)
+    if args.uses:
+        print(uses)
+        
+    if args.steps:
+        print(steps)
 
-        if args.uses:
-            print(uses)
 
-        if args.steps:
-            print(steps)
-
-if args.web:
-    print('<table>')
-    print('    <th>Vertex AI Feature</th>')
-    print('    <th>Description</th>')
-    print('    <th>Open in</th>')
-    
 if args.notebook_dir:
     if not os.path.isdir(args.notebook_dir):
         print("Error: not a directory:", args.notebook_dir)
         exit(1)
-    tag = ''
     parse_dir(args.notebook_dir)
 elif args.notebook:
     if not os.path.isfile(args.notebook):
         print("Error: not a notebook:", args.notebook)
         exit(1)
-    tag = ''
     parse_notebook(args.notebook)
-elif args.notebook_file:
-    if not os.path.isfile(args.notebook_file):
-        print("Error: file does not exist", args.notebook_file)
-    else:
-        last_tag = ''
-        with open(args.notebook_file, 'r') as csvfile:
-            reader = csv.reader(csvfile)
-            heading = True
-            for row in reader:
-                if heading:
-                    heading = False
-                else:
-                    tag = row[0]
-                    notebook = row[1]
-                    try:
-                        linkback = row[2]
-                    except:
-                        linkback = None
-                    parse_notebook(notebook)
 else:
-    print("Error: must specify a directory or notebook")
-    exit(1)
-
-if args.web:
-    print('</table>\n')
+        print("Error: must specify a directory or notebook")
+        exit(1)

notebooks/official/CODEOWNERS

@@ -16,7 +16,6 @@
 /model_monitoring @andrewferlitsch
 /tensorboard @zbl94
 
-/bigquery_ml/bqml-online-prediction.ipynb @polong-lin
 /model_monitoring/model_monitoring.ipynb @mco-gh
 /ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb @jialuzh
 /ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb @jialuzh
@@ -29,11 +28,6 @@
 /automl/automl_forecasting_bqml_arima_plus_comparison.ipynb @TheMichaelHu
 /automl/automl_tabular_on_vertex_pipelines.ipynb @helinwang
 /custom/custom_training_tensorboard_profiler.ipynb @itseric
-/workbench/spark/spark_sample_notebook.ipynb @bradmiro
-/workbench/spark/spark_ml.ipynb @bradmiro
+/workbench/spark/spark_sample_notebook.ipynb @bmiro
 /model-registry/bqml-vertexai-model-registry.ipynb @soheilazangeneh
 /workbench/exploratory_data_analysis/explore_data_in_bigquery_with_workbench.ipynb @alokpattani
-/model_evaluation/automl_tabular_classification_model_evaluation.ipynb @soheilazangeneh
-/model_evaluation/automl_tabular_regression_model_evaluation.ipynb @soheilazangeneh
-/tabular_workflows/tabnet_on_vertex_pipelines.ipynb @sakagarwal
-/tabular_workflows/wide_and_deep_on_vertex_pipelines.ipynb @sakagarwal

notebooks/official/automl/automl-text-classification.ipynb

@@ -52,7 +52,7 @@
         "    </a>\n",
         "  </td>\n",
         " <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/automl/automl-text-classification.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl-text-classification.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -714,7 +714,8 @@
         "deployed_model_display_name = f\"e2e-deployed-text-classification-model-{TIMESTAMP}\"\n",
         "\n",
         "endpoint = model.deploy(\n",
-        "    deployed_model_display_name=deployed_model_display_name, sync=True\n",
+        "    deployed_model_display_name=deployed_model_display_name, \n",
+        "    sync=True\n",
         ")"
       ]
     },

notebooks/official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb

@@ -29,16 +29,16 @@
         "id": "title"
       },
       "source": [
-        "# Compare Vertex AI Forecasting and BigQuery ML ARIMA_PLUS\n",
+        "# Compare Vertex Forecasting and BQML ARIMA_PLUS\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
@@ -61,28 +61,7 @@
       "source": [
         "## Overview\n",
         "\n",
-        "In this tutorial, you take on the role of a store planner who must determine how much inventory they will need to order for each of their products and stores for November 2019. You will accomplish this by training forecasting models using historical sales data. You will start with a baseline model using BigQuery ML (BQML) [ARIMA_PLUS](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-create-time-series) and then compare it against a [Vertex AI Forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/overview) model."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "objective:automl,training,batch_prediction"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "In this tutorial, you learn how to create an BQML ARIMA_PLUS model using a training [Vertex AI Pipeline](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) from [Google Cloud Pipeline Components](https://cloud.google.com/vertex-ai/docs/pipelines/components-introduction) (GCPC), and then do a batch prediction using the corresponding prediction pipeline. You then train a Vertex AI Forecasting model using the same data and compare the evaluation metrics.\n",
-        "\n",
-        "The steps performed are:\n",
-        "\n",
-        "- Train the BQML ARIMA_PLUS model.\n",
-        "- View BQML model evaluation.\n",
-        "- Make a batch prediction with the BQML model.\n",
-        "- Create a Vertex AI `Dataset` resource.\n",
-        "- Train the Vertex AI Forecasting model.\n",
-        "- View the Model evaluation.\n",
-        "- Make a batch prediction with the Model.\n"
+        "In this tutorial, you will take on the role of a store planner who must determine how much inventory they will need to order for each of their products and stores for November 2019. You will accomplish this by training forecasting models using historical sales data. You will start with a baseline model using [BQML ARIMA_PLUS](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-create-time-series) and then compare it against a [Vertex Forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/overview) model."
       ]
     },
     {
@@ -93,7 +72,28 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "To demonstrate the tradeoffs between using BQML and Vertex AI Forecasting, this tutorial will use a synthetic dataset where product sales are dependent on a variety of factors such as advertisements, holidays, and locations. You will see how well a univariate model like ARIMA_PLUS can forecast future sales without knowing information about these factors explicitly, and how well a multivariate model like Vertex AI Forecasting can perform when these factors are known."
+        "To demonstrate the tradeoffs between using BQML and Vertex Forecasting, this tutorial will use a synthetic dataset where product sales are dependent on a variety of factors such as advertisements, holidays, and locations. You will see how well a univariate model like ARIMA_PLUS can forecast future sales without knowing information about these factors explicitly, and how well a multivariate model like Vertex Forecasting can perform when these factors are known."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "objective:automl,training,batch_prediction"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you create an BQML ARIMA_PLUS model using a training [Vertex Pipeline](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) from [Google Cloud Pipeline Components](https://cloud.google.com/vertex-ai/docs/pipelines/components-introduction) (GCPC), and then do a batch prediction using the corresponding prediction pipeline. You then train a Vertex Forecasting model using the same data and compare the evaluation metrics.\n",
+        "\n",
+        "The steps performed are:\n",
+        "\n",
+        "- Train the BQML ARIMA_PLUS model.\n",
+        "- View BQML model evaluation.\n",
+        "- Make a batch prediction with the BQML model.\n",
+        "- Create a Vertex `Dataset` resource.\n",
+        "- Train the Vertex Forecasting model.\n",
+        "- View the Model evaluation.\n",
+        "- Make a batch prediction with the Model.\n"
       ]
     },
     {
@@ -155,9 +155,9 @@
         "id": "install_aip:mbsdk"
       },
       "source": [
-        "## Installation\n",
+        "## Install additional packages\n",
         "\n",
-        "Install the following packages required to execute this notebook."
+        "Install the latest version of the Google Cloud Pipeline Components (GCPC) SDK."
       ]
     },
     {
@@ -170,14 +170,8 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "# Google Cloud Notebook\n",
+        "if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    USER_FLAG = \"--user\"\n",
         "else:\n",
         "    USER_FLAG = \"\"\n",
@@ -185,7 +179,7 @@
         "! (pip3 install --upgrade $USER_FLAG \\\n",
         "    google-cloud-bigquery[pandas]==2.34.4 \\\n",
         "    google-cloud-aiplatform==1.16.1 \\\n",
-        "    google-cloud-pipeline-components==1.0.23)"
+        "    google-cloud-pipeline-components==1.0.18)"
       ]
     },
     {
@@ -246,7 +240,7 @@
         "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
         "\n",
-        "6. (optional) You may also specify a service account to use to run Vertex AI Pipelines in the project.\n",
+        "6. (optional) You may also specify a service account to use to run Vertex Pipelines in the project.\n",
         "\n",
         "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
       ]
@@ -310,7 +304,7 @@
         "#### Region\n",
         "All BigQuery operations (`DATA_REGION`) are set to run in the `US` multi-region. This is required by the ARIMA pipeline because the data you will be using is stored in this region. All destination tables will also be stored in this region.\n",
         "\n",
-        "You may change the `REGION` variable, which is used for Vertex AI Forecasting operations\n",
+        "You may change the `REGION` variable, which is used for Vertex Forecasting operations\n",
         "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
         "\n",
         "- Americas: `us-central1`\n",
@@ -330,11 +324,8 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
         "DATA_REGION = \"US\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -343,9 +334,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### UUID\n",
+        "#### Timestamp\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -356,16 +347,9 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
@@ -376,7 +360,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated.\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -411,11 +395,8 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebook, then don't execute this code\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -423,9 +404,10 @@
         "\n",
         "    # If you are running this notebook locally, replace the string below with the\n",
         "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
+        "    # account. Alternatively, you may edit this notebook to authenticate using\n",
+        "    # gcloud.\n",
         "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS '[your-service-account-key-path]'"
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
       ]
     },
     {
@@ -438,7 +420,7 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "When you initialize the Vertex AI SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
+        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
         "\n",
         "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
       ]
@@ -472,9 +454,9 @@
       "outputs": [],
       "source": [
         "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP\n",
         "\n",
-        "! gsutil ls -b $BUCKET_URI || gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil ls -b $BUCKET_URI || gsutil mb -l $DATA_REGION $BUCKET_URI"
       ]
     },
     {
@@ -532,9 +514,9 @@
         "id": "init_aip:mbsdk"
       },
       "source": [
-        "## Initialize Vertex AI SDK for Python\n",
+        "## Initialize Vertex SDK for Python\n",
         "\n",
-        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
+        "Initialize the Vertex SDK for Python for your project and corresponding bucket."
       ]
     },
     {
@@ -576,8 +558,8 @@
       },
       "outputs": [],
       "source": [
-        "arima_dataset_name = f\"forecasting_demo_arima_{UUID}\"\n",
-        "vertex_dataset_name = f\"forecasting_demo_vertex_{UUID}\"\n",
+        "arima_dataset_name = f\"forecasting_demo_arima_{TIMESTAMP}\"\n",
+        "vertex_dataset_name = f\"forecasting_demo_vertex_{TIMESTAMP}\"\n",
         "\n",
         "arima_dataset_path = \".\".join([PROJECT_ID, arima_dataset_name])\n",
         "vertex_dataset_path = \".\".join([PROJECT_ID, vertex_dataset_name])\n",
@@ -810,15 +792,15 @@
         "\n",
         "Now you are ready to start creating your own BQML ARIMA_PLUS model.\n",
         "\n",
-        "Like with Vertex AI Forecasting, the pipeline you will run will train evaluation models using the training and validation sets and use backtesting to create evaluation metrics on the test set. Finally, a serving model will be produced that uses all available data.\n",
+        "Like with Vertex Forecasting, the pipeline you will run will train evaluation models using the training and validation sets and use backtesting to create evaluation metrics on the test set. Finally, a serving model will be produced that uses all available data.\n",
         "\n",
         "**How do you estimate the cost?**\n",
         "\n",
-        "Backtesting involves training a single BQML model for each period in the test set, so the cost is a function of the length of the test set after any downsampling done by the windowing strategy. The cost is also multiplied by the number of candidate models trained, which is determined by `max_order`.\n",
+        "Backtesting will involve training a single BQML model for each period in the test set, so the cost will be a function of the length of the test set. The cost is also multiplied by the number of candidate models trained, which is determined by `max_order`.\n",
         "\n",
-        "According to [BQ pricing](https://cloud.google.com/bigquery-ml/pricing), BQML model creation costs $250 per TB. We'll use a max order of 3, which translates to 20 candidate models when there are multiple time series. Our demo dataset is 3 MB in size, and includes 31 test periods. We window with a stride length of 1, so all periods are used for evaluation.\n",
+        "According to [BQ pricing](https://cloud.google.com/bigquery-ml/pricing), BQML model creation costs $250 per TB. We'll use a max order of 3, which translates to 20 candidate models when there are multiple time series. Our demo dataset is 3 MB in size, and includes 31 test periods.\n",
         "\n",
-        "In this tutorial, the model create stage of the pipeline costs `3 MB * ($250 / 1024^2) * (31 / 1) periods * 20 candidates = $0.44`."
+        "In this tutorial, the model create stage of the pipeline costs `3 MB * ($250 / 1024^2) * 31 periods * 20 candidates = $0.44`."
       ]
     },
     {
@@ -839,23 +821,47 @@
         "\n",
         "- `bigquery_destination_uri`: (optional) BigQuery Dataset URI. Used to export the metrics table and model. If not given, we will create one for the user.\n",
         "- `data_granularity_unit`: Enum used to specify the time granularity (hour, day, week, month, etc).\n",
-        "- `data_source_csv_filenames` or `data_source_bigquery_table_path`: A URI for either a CSV stored in GCR or a BigQuery table, respectively.\n",
+        "- `data_source`: JSON for specifying the input data source URI and type. Similar to a Vertex Dataset. Currently supports BigQuery and CSV (in GCS) data sources.\n",
+        "\n",
+        "  It can look like either:\n",
+        "  ```json\n",
+        "  {\n",
+        "    \"big_query_data_source\": {\n",
+        "      \"big_query_table_path\": \"bq://[PROJECT].[DATASET].[TABLE]\"\n",
+        "    }\n",
+        "  }\n",
+        "  ```\n",
+        "  or\n",
+        "  ```json\n",
+        "  {\n",
+        "    \"csv_data_source\": {\n",
+        "      \"csv_filenames\": [ [GCS_PATHS] ],\n",
+        "  }\n",
+        "  ```\n",
         "- `evaluated_examples_destination_uri\t`: (optional) BigQuery Dataset URI OR Table URI. Used to export the evaluated examples table. Will use bigquery_destination_uri if not provided.\n",
         "- `forecast_horizon`: Integer number of periods to predict.\n",
-        "- A data splitting strategy of either:\n",
-        "  - `predefined_split_key`: A column containing `TRAIN`, `VALIDATE`, or `TEST` to denote the splits for each row.\n",
-        "  -  `training_fraction`, `validation_fraction`, and `test_fraction` to set the fractions to split on chronologically on the time column.\n",
-        "  - `timestamp_split_key` plus the fractions in the previous option to perform fractional splitting on a column other than the time column.\n",
-        "- A windowing strategy of either:\n",
-        "  - `window_column`: A boolean column decides whether or now each row gets considered when calculating the evaluation metrics.\n",
-        "  - `window_stride_length`: Every N rows will be used to compute the evaluation metrics.\n",
-        "  - `window_max_count`: Downsample rows such that only the given number are used to calculate the evaluation metrics.\n",
-        "- `target_column`: Name of target column.\n",
+        "- `split_spec`: JSON for specifying how data should be split. Supports predefined split and fractional split.\n",
+        "  \n",
+        "  It can look like either:\n",
+        "  ```json\n",
+        "  {\"predefined_split\": {\"key\": \"[SPLIT_COLUMN]\"}}\n",
+        "  ```\n",
+        "  or\n",
+        "  ```json\n",
+        "  {\n",
+        "    \"fraction_split\": {\n",
+        "        \"training_fraction\": 0.8,\n",
+        "        \"validation_fraction\": 0.1,\n",
+        "        \"test_fraction\": 0.1,\n",
+        "    },\n",
+        "  }\n",
+        "  ```\n",
+        "- `target_column_name`: Name of target column.\n",
         "- `time_column`: Name of time column.\n",
         "- `time_series_identifier_column`: Name of id column.\n",
         "- `max_order`: Integer between 1 and 5 representing the size of the parameter search space for ARIMA_PLUS. 5 would result in the highest accuracy model, but also the longest training runtime/cost.\n",
         "\n",
-        "The execution of the training pipeline may take around **20 minutes**."
+        "The execution of the training pipeline will take around **20 minutes**."
       ]
     },
     {
@@ -872,24 +878,31 @@
         "forecast_horizon = 30  # @param {type: \"integer\"}\n",
         "data_granularity_unit = \"day\"  # @param {type: \"string\"}\n",
         "split_column = \"split\"  # @param {type: \"string\"}\n",
-        "window_stride_length = 1  # @param {type: \"integer\"}\n",
         "max_order = 3  # @param {type: \"integer\"}\n",
         "override_destination = True  # @param {type: \"boolean\"}\n",
         "\n",
+        "split_spec = {\"predefined_split\": {\"key\": split_column}}\n",
+        "data_source = {\n",
+        "    \"big_query_data_source\": {\n",
+        "        \"big_query_table_path\": TRAINING_DATASET_BQ_PATH,\n",
+        "    },\n",
+        "}\n",
+        "window_config = {\"stride\": 1}\n",
+        "\n",
         "(\n",
         "    train_job_spec_path,\n",
         "    train_parameter_values,\n",
         ") = utils.get_bqml_arima_train_pipeline_and_parameters(\n",
         "    project=PROJECT_ID,\n",
-        "    location=REGION,\n",
+        "    location=DATA_REGION,\n",
         "    time_column=time_column,\n",
         "    time_series_identifier_column=time_series_identifier_column,\n",
-        "    target_column=target_column,\n",
+        "    target_column_name=target_column,\n",
         "    forecast_horizon=forecast_horizon,\n",
         "    data_granularity_unit=data_granularity_unit,\n",
-        "    predefined_split_key=split_column,\n",
-        "    data_source_bigquery_table_path=TRAINING_DATASET_BQ_PATH,\n",
-        "    window_stride_length=window_stride_length,\n",
+        "    split_spec=split_spec,\n",
+        "    data_source=data_source,\n",
+        "    window_config=window_config,\n",
         "    bigquery_destination_uri=arima_dataset_path,\n",
         "    override_destination=override_destination,\n",
         "    max_order=max_order,\n",
@@ -904,9 +917,9 @@
       "source": [
         "### Run the training pipeline\n",
         "\n",
-        "Use the Vertex AI Python SDK to kick off a training pipeline run. Once the run has started, the following cell will output a link that will allow you to monitor the run. The link should look like this: \n",
+        "Use the Vertex Python SDK to kick off a training pipeline run. Once the run has started, the following cell will output a link that will allow you to monitor the run. The link should look like this: \n",
         "\n",
-        "`https://console.cloud.google.com/vertex-ai/locations/[REGION]/pipelines/runs/[DISPLAY_NAME]`"
+        "`https://console.cloud.google.com/vertex-ai/locations/[DATA_REGION]/pipelines/runs/[DISPLAY_NAME]`"
       ]
     },
     {
@@ -918,7 +931,8 @@
       "outputs": [],
       "source": [
         "# The display name should be unique even if this cell is rerun.\n",
-        "DISPLAY_NAME = f\"forecasting-demo-train-{generate_uuid()}\"\n",
+        "now = datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
+        "DISPLAY_NAME = f\"forecasting-demo-train-{now}\"\n",
         "\n",
         "job = aiplatform.PipelineJob(\n",
         "    job_id=DISPLAY_NAME,\n",
@@ -989,11 +1003,27 @@
         "Now that your Model resource is trained, you can make a batch prediction using the prediction pipeline, with the following parameters:\n",
         "\n",
         "- `bigquery_destination_uri`: (optional) BigQuery Dataset URI. Used to export the metrics table and model. If not given, we will create one for the user.\n",
-        "- `data_source_csv_filenames` or `data_source_bigquery_table_path`: A URI for either a CSV stored in GCR or a BigQuery table, respectively.\n",
+        "- `data_source`: JSON for specifying the input data source URI and type. Similar to a Vertex Dataset. Currently supports BigQuery and CSV (in GCS) data sources.\n",
+        "\n",
+        "  It can look like either:\n",
+        "  ```json\n",
+        "  {\n",
+        "    \"big_query_data_source\": {\n",
+        "      \"big_query_table_path\": \"bq://[PROJECT].[DATASET].[TABLE]\"\n",
+        "    }\n",
+        "  }\n",
+        "  ```\n",
+        "  or\n",
+        "  ```json\n",
+        "  {\n",
+        "    \"csv_data_source\": {\n",
+        "      \"csv_filenames\": [ [GCS_PATHS] ],\n",
+        "  }\n",
+        "  ```\n",
         "- `generate_explanation`: If True, the predictions table will have some extra xAI columns.\n",
         "- `model_name`: Name of an existing BQML ARIMA_PLUS model to use for predictions.\n",
         "\n",
-        "The execution of the prediction pipeline may take around **5 minutes**."
+        "The execution of the prediction pipeline will take around **5 minutes**."
       ]
     },
     {
@@ -1004,8 +1034,14 @@
       },
       "outputs": [],
       "source": [
+        "data_source = {\n",
+        "    \"big_query_data_source\": {\n",
+        "        \"big_query_table_path\": PREDICTION_DATASET_BQ_PATH,\n",
+        "    },\n",
+        "}\n",
+        "\n",
         "# Get the model name programmatically, you can also find this by looking at the\n",
-        "# execution graph in Vertex AI Pipelines.\n",
+        "# execution graph in Vertex Pipelines.\n",
         "for task_detail in job.gca_resource.job_detail.task_details:\n",
         "    if task_detail.task_name == \"bigquery-create-model-job\":\n",
         "        model_name = task_detail.outputs[\"model\"].artifacts[0].metadata[\"modelId\"]\n",
@@ -1019,9 +1055,9 @@
         "    predict_parameter_values,\n",
         ") = utils.get_bqml_arima_predict_pipeline_and_parameters(\n",
         "    project=PROJECT_ID,\n",
-        "    location=REGION,\n",
+        "    location=DATA_REGION,\n",
         "    model_name=f\"{arima_dataset_path}.{model_name}\",\n",
-        "    data_source_bigquery_table_path=PREDICTION_DATASET_BQ_PATH,\n",
+        "    data_source=data_source,\n",
         "    bigquery_destination_uri=arima_dataset_path,\n",
         ")"
       ]
@@ -1034,9 +1070,9 @@
       "source": [
         "### Run the prediction pipeline\n",
         "\n",
-        "Use the Vertex AI Python SDK to kick off a prediction pipeline run. Once the run has started, the following cell will output a link that will allow you to monitor the run. The link should look like this: \n",
+        "Use the Vertex Python SDK to kick off a prediction pipeline run. Once the run has started, the following cell will output a link that will allow you to monitor the run. The link should look like this: \n",
         "\n",
-        "`https://console.cloud.google.com/vertex-ai/locations/[REGION]/pipelines/runs/[DISPLAY_NAME]`"
+        "`https://console.cloud.google.com/vertex-ai/locations/[DATA_REGION]/pipelines/runs/[DISPLAY_NAME]`"
       ]
     },
     {
@@ -1048,7 +1084,8 @@
       "outputs": [],
       "source": [
         "# The display name should be unique even if this cell is rerun.\n",
-        "DISPLAY_NAME = f\"forecasting-demo-predict-{generate_uuid()}\"\n",
+        "now = datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
+        "DISPLAY_NAME = f\"forecasting-demo-predict-{now}\"\n",
         "\n",
         "job = aiplatform.PipelineJob(\n",
         "    job_id=DISPLAY_NAME,\n",
@@ -1080,7 +1117,7 @@
       "outputs": [],
       "source": [
         "# Get the prediction table programmatically, you can also find this by looking at the\n",
-        "# execution graph in Vertex AI Pipelines.\n",
+        "# execution graph in Vertex Pipelines.\n",
         "for task_detail in job.gca_resource.job_detail.task_details:\n",
         "    if task_detail.task_name == \"bigquery-query-job\":\n",
         "        pred_table = (\n",
@@ -1205,7 +1242,7 @@
         "id": "59qKXL9ARO97"
       },
       "source": [
-        "# Compare Against Vertex AI Forecasting"
+        "# Compare Against Vertex Forecasting"
       ]
     },
     {
@@ -1234,7 +1271,7 @@
       "outputs": [],
       "source": [
         "dataset = aiplatform.TimeSeriesDataset.create(\n",
-        "    display_name=\"forecasting_demo_train\" + \"_\" + UUID,\n",
+        "    display_name=\"forecasting_demo_train\" + \"_\" + TIMESTAMP,\n",
         "    bq_source=[TRAINING_DATASET_BQ_PATH],\n",
         ")\n",
         "print(dataset.resource_name)"
@@ -1280,12 +1317,11 @@
         "    \"product\": \"categorical\",\n",
         "    \"holiday\": \"categorical\",\n",
         "}\n",
-        "available_at_forecast_columns_ = [\n",
+        "available_at_forecast_columns = [\n",
         "    \"date\",\n",
         "    \"advertisement\",\n",
         "    \"holiday\",\n",
-        "]\n",
-        "available_at_forecast_columns = available_at_forecast_columns_  # @param {type: \"raw\"}\n",
+        "]  # @param {type: \"raw\"}\n",
         "unavailable_at_forecast_columns = [\"sales\"]  # @param {type: \"raw\"}\n",
         "time_series_attribute_columns = [\"store\", \"product\"]  # @param {type: \"raw\"}\n",
         "context_window = 30  # @param {type: \"integer\"}\n",
@@ -1301,7 +1337,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DISPLAY_NAME = f\"forecasting-demo-model_{UUID}\"\n",
+        "MODEL_DISPLAY_NAME = f\"forecasting-demo-model_{TIMESTAMP}\"\n",
         "\n",
         "training_job = aiplatform.AutoMLForecastingTrainingJob(\n",
         "    display_name=MODEL_DISPLAY_NAME,\n",
@@ -1337,7 +1373,7 @@
         "\n",
         "The `run` method, when completed, returns the `Model` resource.\n",
         "\n",
-        "The execution of the training pipeline may take up to **one hour**. You can learn about the pricing for Vertex AI Forecasting [here](https://cloud.google.com/vertex-ai/pricing#tabular-data)."
+        "The execution of the training pipeline will take up to **one hour**. You can learn about the pricing for Vertex Forecasting [here](https://cloud.google.com/vertex-ai/pricing#tabular-data)."
       ]
     },
     {
@@ -1364,7 +1400,6 @@
         "    budget_milli_node_hours=budget_milli_node_hours,\n",
         "    model_display_name=MODEL_DISPLAY_NAME,\n",
         "    predefined_split_column_name=split_column,\n",
-        "    window_stride_length=window_stride_length,\n",
         ")"
       ]
     },
@@ -1421,7 +1456,7 @@
         "\n",
         "Now that you have backtesting metrics from both models, you can compare the two side-by-side.\n",
         "\n",
-        "Since the sales in this dataset were a function of covariates, we should expect the MAE, RMSE, and MAPE to be lower when using Vertex AI Forecasting. The BQML ARIMA_PLUS evaluation metrics show the relative impact of including these additional features in a model."
+        "Since the sales in this dataset were a function of covariates, we should expect the MAE, RMSE, and MAPE to be lower when using Vertex Forecasting. The BQML ARIMA_PLUS evaluation metrics show the relative impact of including these additional features in a model."
       ]
     },
     {
@@ -1443,7 +1478,7 @@
       "source": [
         "## Send a batch prediction request\n",
         "\n",
-        "The following section shows how you can send a batch prediction to your Vertex AI Forecasting model in case you want to compare the models at serving time."
+        "The following section shows how you can send a batch prediction to your Vertex Forecasting model in case you want to compare the models at serving time."
       ]
     },
     {
@@ -1475,7 +1510,7 @@
       "outputs": [],
       "source": [
         "batch_prediction_job = model.batch_predict(\n",
-        "    job_display_name=f\"forecasting_demo_predictions_{UUID}\",\n",
+        "    job_display_name=f\"forecasting_demo_predictions_{TIMESTAMP}\",\n",
         "    bigquery_source=PREDICTION_DATASET_BQ_PATH,\n",
         "    instances_format=\"bigquery\",\n",
         "    bigquery_destination_prefix=f\"bq://{vertex_dataset_path}\",\n",
@@ -1496,7 +1531,7 @@
         "\n",
         "Next, wait for the batch job to complete. Alternatively, you can set the parameter `sync` to `True` in the `batch_predict()` method to block until the batch prediction job is completed.\n",
         "\n",
-        "The execution of the prediction pipeline may take up to 30 minutes.\n"
+        "The execution of the prediction pipeline will take up to 30 minutes.\n"
       ]
     },
     {
@@ -1563,7 +1598,7 @@
       },
       "outputs": [],
       "source": [
-        "print(\"Click the link below to view Vertex AI Forecasting predictions:\")\n",
+        "print(\"Click the link below to view Vertex Forecasting predictions:\")\n",
         "print(\n",
         "    get_data_studio_link(\n",
         "        batch_prediction_bq_input_uri=actuals_table,\n",
@@ -1581,7 +1616,7 @@
         "id": "cleanup:mbsdk"
       },
       "source": [
-        "## Clean up Vertex AI and BigQuery resources\n",
+        "## Clean up Vertex and BigQuery resources\n",
         "\n",
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",

notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb

@@ -33,12 +33,12 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
@@ -855,7 +855,6 @@
         "    run_distillation=run_distillation,\n",
         "    dataflow_subnetwork=dataflow_subnetwork,\n",
         "    dataflow_use_public_ips=dataflow_use_public_ips,\n",
-        "    export_additional_model_without_custom_ops=export_additional_model_without_custom_ops,\n",
         ")\n",
         "\n",
         "job_id = \"automl-tabular-{}\".format(uuid.uuid4())\n",

notebooks/official/automl/sdk_automl_tabular_forecasting_batch.ipynb

@@ -44,7 +44,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/automl/sdk_automl_tabular_forecasting_batch.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_forecasting_batch.ipynb\">\n",
         "        <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -68,12 +68,23 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "02b9af111927"
+        "id": "dataset:covid,forecast"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial a time series dataset containing samples drawn from the Iowa Liquor Retail Sales dataset. Data were made available by the Iowa Department of Commerce. It is provided under the Creative Commons Zero v1.0 Universal license. For more details, see: https://console.cloud.google.com/marketplace/product/iowa-department-of-commerce/iowa-liquor-sales. This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in BigQuery."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "objective:automl,training,batch_prediction"
       },
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to create an `AutoML` tabular forecasting model from a Python script, and then do a batch prediction using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
+        "In this tutorial, you create an `AutoML` tabular forecasting model from a Python script, and then do a batch prediction using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
         "\n",
         "This tutorial uses the following Google Cloud ML services:\n",
         "\n",
@@ -86,18 +97,7 @@
         "- Create a `Vertex AI Dataset` resource.\n",
         "- Train an `AutoML` tabular forecasting `Model` resource.\n",
         "- Obtain the evaluation metrics for the `Model` resource.\n",
-        "- Make a batch prediction."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:covid,forecast"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is a time series dataset containing samples drawn from the Iowa Liquor Retail Sales dataset. Data is made available by the Iowa Department of Commerce. It is provided under the Creative Commons Zero v1.0 Universal license. For more details, see: https://console.cloud.google.com/marketplace/product/iowa-department-of-commerce/iowa-liquor-sales. This dataset does not require any feature engineering. The version of the dataset you use in this tutorial is stored in BigQuery."
+        "- Make a batch prediction.\n"
       ]
     },
     {
@@ -128,38 +128,29 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step.\n",
+        "If you are using Colab or Workbench AI Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
         "\n",
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
+        "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
         "\n",
-        "* The Google Cloud SDK\n",
-        "* Git\n",
-        "* Python 3\n",
-        "* virtualenv\n",
-        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "- The Cloud Storage SDK\n",
+        "- Git\n",
+        "- Python 3\n",
+        "- virtualenv\n",
+        "- Jupyter notebook running in a virtual environment with Python 3\n",
         "\n",
-        "The Google Cloud guide to [Setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
+        "The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:\n",
         "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
+        "1. [Install and initialize the SDK](https://cloud.google.com/sdk/docs/).\n",
         "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
+        "2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).\n",
         "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3.  Activate the virtual environment.\n",
         "\n",
-        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
+        "4. To install Jupyter, run `pip3 install jupyter` on the command-line in a terminal shell.\n",
         "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "5. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
         "\n",
-        "1. Open this notebook in the Jupyter Notebook Dashboard."
+        "6. Open this notebook in the Jupyter Notebook Dashboard.\n"
       ]
     },
     {
@@ -170,7 +161,7 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the following packages required to execute this notebook. "
+        "Install the latest version of Vertex AI SDK for Python."
       ]
     },
     {
@@ -184,7 +175,7 @@
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",
@@ -194,7 +185,8 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q"
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
+        "! pip3 install --upgrade tensorflow $USER_FLAG -q"
       ]
     },
     {
@@ -205,7 +197,7 @@
       "source": [
         "### Restart the kernel\n",
         "\n",
-        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
       ]
     },
     {
@@ -216,7 +208,6 @@
       },
       "outputs": [],
       "source": [
-        "# Automatically restart kernel after installs\n",
         "import os\n",
         "\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
@@ -227,48 +218,34 @@
         "    app.kernel.do_shutdown(True)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "0e3cab0cc491"
-      },
-      "source": [
-        "## Before you begin"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "before_you_begin:nogpu"
       },
       "source": [
+        "## Before you begin\n",
+        "\n",
+        "### GPU runtime\n",
+        "\n",
+        "This tutorial does not require a GPU runtime.\n",
+        "\n",
         "### Set up your Google Cloud project\n",
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
         "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
         "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
         "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
+        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
         "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
         "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
         "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "1460fd744366"
-      },
-      "source": [
-        "#### Set your project ID\n",
-        "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
       ]
     },
     {
@@ -317,7 +294,7 @@
         "#### Region\n",
         "\n",
         "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
         "\n",
         "- Americas: `us-central1`\n",
         "- Europe: `europe-west4`\n",
@@ -325,7 +302,7 @@
         "\n",
         "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
         "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
       ]
     },
     {
@@ -348,9 +325,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### UUID\n",
+        "#### Timestamp\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -361,16 +338,9 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
@@ -381,31 +351,23 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step.\n",
+        "**If you are using Workbench AI Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
         "**Otherwise**, follow these steps:\n",
         "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
         "\n",
-        "2. Click **Create service account**.\n",
+        "**Click Create service account**.\n",
         "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
+        "In the **Service account name** field, enter a name, and click **Create**.\n",
         "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
         "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
+        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
         "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell.\n"
+        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
       ]
     },
     {
@@ -476,9 +438,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
     {
@@ -498,7 +460,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -527,6 +489,9 @@
         "id": "setup_vars"
       },
       "source": [
+        "### Set up variables\n",
+        "\n",
+        "Next, set up some variables used throughout the tutorial.\n",
         "### Import libraries and define constants"
       ]
     },
@@ -538,10 +503,7 @@
       },
       "outputs": [],
       "source": [
-        "import urllib\n",
-        "\n",
-        "import google.cloud.aiplatform as aiplatform\n",
-        "from google.cloud import bigquery"
+        "import google.cloud.aiplatform as aiplatform"
       ]
     },
     {
@@ -627,7 +589,7 @@
       "outputs": [],
       "source": [
         "dataset = aiplatform.TimeSeriesDataset.create(\n",
-        "    display_name=\"iowa_liquor_sales_train\" + \"_\" + UUID,\n",
+        "    display_name=\"iowa_liquor_sales_train\" + \"_\" + TIMESTAMP,\n",
         "    bq_source=[TRAINING_DATASET_BQ_PATH],\n",
         ")\n",
         "\n",
@@ -687,7 +649,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DISPLAY_NAME = f\"iowa-liquor-sales-forecast-model_{UUID}\"\n",
+        "MODEL_DISPLAY_NAME = f\"iowa-liquor-sales-forecast-model_{TIMESTAMP}\"\n",
         "\n",
         "training_job = aiplatform.AutoMLForecastingTrainingJob(\n",
         "    display_name=MODEL_DISPLAY_NAME,\n",
@@ -810,7 +772,11 @@
       },
       "outputs": [],
       "source": [
-        "batch_predict_bq_output_dataset_name = f\"iowa_liquor_sales_predictions_{UUID}\"\n",
+        "import os\n",
+        "\n",
+        "from google.cloud import bigquery\n",
+        "\n",
+        "batch_predict_bq_output_dataset_name = f\"iowa_liquor_sales_predictions_{TIMESTAMP}\"\n",
         "batch_predict_bq_output_dataset_path = \"{}.{}\".format(\n",
         "    PROJECT_ID, batch_predict_bq_output_dataset_name\n",
         ")\n",
@@ -854,7 +820,7 @@
         ")\n",
         "\n",
         "batch_prediction_job = model.batch_predict(\n",
-        "    job_display_name=f\"iowa_liquor_sales_forecasting_predictions_{UUID}\",\n",
+        "    job_display_name=f\"iowa_liquor_sales_forecasting_predictions_{TIMESTAMP}\",\n",
         "    bigquery_source=PREDICTION_DATASET_BQ_PATH,\n",
         "    instances_format=\"bigquery\",\n",
         "    bigquery_destination_prefix=batch_predict_bq_output_uri_prefix,\n",
@@ -935,6 +901,8 @@
       },
       "outputs": [],
       "source": [
+        "import urllib\n",
+        "\n",
         "tables = client.list_tables(batch_predict_bq_output_dataset_path)\n",
         "\n",
         "prediction_table_id = \"\"\n",
@@ -1044,6 +1012,9 @@
       },
       "outputs": [],
       "source": [
+        "# Set this to true only if you'd like to delete your bucket\n",
+        "delete_bucket = False\n",
+        "\n",
         "# Delete dataset\n",
         "dataset.delete()\n",
         "\n",
@@ -1056,9 +1027,6 @@
         "# Delete batch prediction job\n",
         "batch_prediction_job.delete()\n",
         "\n",
-        "# Set this to true only if you'd like to delete your bucket\n",
-        "delete_bucket = False\n",
-        "\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"
       ]

notebooks/official/automl/sdk_automl_tabular_regression_batch_bq.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2022 Google LLC.\n",
+        "# Copyright 2022 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",

notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb

@@ -263,7 +263,7 @@
         "\n",
         "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "3. [Enable the following APIs: Vertex AI API, Cloud Resource Manager API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,cloudresourcemanager.googleapis.com).\n",
+        "3. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
         "\n",
         "4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",

notebooks/official/custom/sdk-custom-image-classification-batch.ipynb

@@ -44,7 +44,7 @@
         "    </a>\n",
         "  </td>\n",
         " <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/custom/sdk-custom-image-classification-batch.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official/automl/automl-text-classification.ipynb\">\n",
         "        <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -65,6 +65,17 @@
         "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy a custom image classification model for batch prediction."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:custom,cifar10,icn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [cifar10 dataset](https://www.tensorflow.org/datasets/catalog/cifar10) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you will use is built into TensorFlow. The trained model predicts which type of class an image is from ten classes: airplane, automobile, bird, cat, deer, dog, frog, horse, ship, truck."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -92,17 +103,6 @@
         "- Make a batch prediction."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:custom,cifar10,icn"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [cifar10 dataset](https://www.tensorflow.org/datasets/catalog/cifar10) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you will use is built into TensorFlow. The trained model predicts which type of class an image is from ten classes: airplane, automobile, bird, cat, deer, dog, frog, horse, ship, truck."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/official/custom/sdk-custom-image-classification-online.ipynb

@@ -65,6 +65,17 @@
         "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy a custom image classification model for online prediction."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:custom,cifar10,icn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [cifar10 dataset](https://www.tensorflow.org/datasets/catalog/cifar10) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you will use is built into TensorFlow. The trained model predicts which type of class an image is from ten classes: airplane, automobile, bird, cat, deer, dog, frog, horse, ship, truck."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -92,17 +103,6 @@
         "- Undeploy the `Model` resource."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:custom,cifar10,icn"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [cifar10 dataset](https://www.tensorflow.org/datasets/catalog/cifar10) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). The version of the dataset you will use is built into TensorFlow. The trained model predicts which type of class an image is from ten classes: airplane, automobile, bird, cat, deer, dog, frog, horse, ship, truck."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb

@@ -29,8 +29,6 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# Build Vertex AI Experiment lineage for custom training\n",
-        "\n",
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
@@ -45,7 +43,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",

notebooks/official/experiments/comparing_pipeline_runs.ipynb

@@ -3,6 +3,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "ed0fca3f",
       "metadata": {
         "id": "ur8xi4C7S06n"
       },
@@ -25,12 +26,21 @@
     },
     {
       "cell_type": "markdown",
+      "id": "6e92def7-b4f0-4100-b981-82972665d19d",
+      "metadata": {
+        "id": "847715f095b5"
+      },
+      "source": [
+        "# Vertex AI: Comparing Pipeline Runs"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "ffada1ce",
       "metadata": {
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# Compare pipeline runs with Vertex AI Experiments\n",
-        "\n",
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
@@ -55,10 +65,13 @@
     },
     {
       "cell_type": "markdown",
+      "id": "d118d181",
       "metadata": {
         "id": "tvgnzT1CKxrO"
       },
       "source": [
+        "# Compare pipeline runs with Vertex AI Experiments\n",
+        "\n",
         "## Overview\n",
         "\n",
         "Depending on the model life cycle of your data science team, you would like to experiment and track training Pipeline runs and its associated parameters. Then, you would to compare runs of these Pipelines to each others in order to figure out which is the best configuration generates the model you will register in the Vertex AI Model Registry."
@@ -66,13 +79,17 @@
     },
     {
       "cell_type": "markdown",
+      "id": "b6201ad0-af42-48fd-b03d-403fd235e268",
       "metadata": {
         "id": "d220917f1302"
       },
       "source": [
         "### Objective\n",
         "\n",
-        "In this notebook, you learn how to use `Vertex AI Experiments` to log a pipeline job and compare different pipeline jobs.\n",
+        "In this notebook, you will learn how to use Vertex AI Experiments to \n",
+        "\n",
+        "*   Log Pipeline Job\n",
+        "*   Compare different Pipeline Jobs\n",
         "\n",
         "The steps covered include:\n",
         "\n",
@@ -84,6 +101,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "cffa7608-f550-4913-8f88-30cbcd525685",
       "metadata": {
         "id": "263933842022"
       },
@@ -95,6 +113,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "b46c0eb6-e65d-4b28-b17d-dc9bf9b08120",
       "metadata": {
         "id": "de76bb18c85b"
       },
@@ -115,6 +134,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "ee1e6851",
       "metadata": {
         "id": "gCuSR8GkAgzl"
       },
@@ -157,6 +177,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "97e5b386",
       "metadata": {
         "id": "i7EUnXsZhAGF"
       },
@@ -169,6 +190,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "5b01540f",
       "metadata": {
         "id": "2b4ef9b72d43"
       },
@@ -193,6 +215,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "c6806ee8",
       "metadata": {
         "id": "hhq5zEbGg0XX"
       },
@@ -205,6 +228,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "584f0887",
       "metadata": {
         "id": "EzrelQZ22IZj"
       },
@@ -223,6 +247,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "bb098c06",
       "metadata": {
         "id": "lWEdiXsJg0XY"
       },
@@ -232,6 +257,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "3706598b",
       "metadata": {
         "id": "BF1j6f9HApxa"
       },
@@ -256,6 +282,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "c7f6a1ea",
       "metadata": {
         "id": "WReHDGG5g0XY"
       },
@@ -268,6 +295,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "6984e874-ae15-4094-9c5b-8d12661645c9",
       "metadata": {
         "id": "3c8049930470"
       },
@@ -279,6 +307,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "ebaef2f9",
       "metadata": {
         "id": "oM1iC_MfAts1"
       },
@@ -296,6 +325,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "d9745b2b-cd37-4a1c-aae7-4cd75a3c126c",
       "metadata": {
         "id": "f2e3c0f2cbfb"
       },
@@ -306,6 +336,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "fff3c4af",
       "metadata": {
         "id": "qJYoRfYng0XZ"
       },
@@ -316,6 +347,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "89a05131",
       "metadata": {
         "id": "riG_qUokg0XZ"
       },
@@ -327,6 +359,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "9aa4ad5f",
       "metadata": {
         "id": "2aa333eca058"
       },
@@ -348,6 +381,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "244d416e",
       "metadata": {
         "id": "d8b34ef9a3d0"
       },
@@ -361,6 +395,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "eed6c3ba",
       "metadata": {
         "id": "126548a06aa1"
       },
@@ -373,6 +408,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "6ef7c7b1",
       "metadata": {
         "id": "e660b8504e63"
       },
@@ -392,6 +428,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "5763da2c",
       "metadata": {
         "id": "sBCra4QMA2wR"
       },
@@ -433,6 +470,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "85b826d2",
       "metadata": {
         "id": "PyQmSRbKA8r-"
       },
@@ -465,6 +503,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "3d31b8b8",
       "metadata": {
         "id": "zgPO1eR3CYjk"
       },
@@ -481,6 +520,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "95a18950",
       "metadata": {
         "id": "MzGDU7TWdts_"
       },
@@ -493,6 +533,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "379d758a",
       "metadata": {
         "id": "cf221059d072"
       },
@@ -505,6 +546,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "50b86adc",
       "metadata": {
         "id": "-EcIXiGsCePi"
       },
@@ -515,6 +557,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "564ce38f",
       "metadata": {
         "id": "NIq7R4HZCfIc"
       },
@@ -525,6 +568,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "a4324d2e",
       "metadata": {
         "id": "ucvCsknMCims"
       },
@@ -535,6 +579,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "df030498-f6e7-4e45-96f4-0d36590865aa",
       "metadata": {
         "id": "vhOb7YnwClBb"
       },
@@ -545,6 +590,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "49d12f54",
       "metadata": {
         "id": "b7e24e522bee"
       },
@@ -557,6 +603,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "8da829a1",
       "metadata": {
         "id": "77b01a1fdbb4"
       },
@@ -568,6 +615,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "f2c8c41d",
       "metadata": {
         "id": "121d7ca29426"
       },
@@ -597,6 +645,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "f0552e61",
       "metadata": {
         "id": "aa175e2960ac"
       },
@@ -609,6 +658,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "abbfcad3",
       "metadata": {
         "id": "f88cb0488c08"
       },
@@ -621,6 +671,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "2689f52b",
       "metadata": {
         "id": "fXUqOdIaLbjf"
       },
@@ -631,6 +682,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "754e5f7b",
       "metadata": {
         "id": "9fYX14c0LfmU"
       },
@@ -643,6 +695,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "231e5499",
       "metadata": {
         "id": "XoEqT2Y4DJmf"
       },
@@ -653,6 +706,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "884b2d31",
       "metadata": {
         "id": "pRUOFELefqf1"
       },
@@ -678,6 +732,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "20e96b6b",
       "metadata": {
         "id": "OAY0QKZD8qNP"
       },
@@ -698,6 +753,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "3fb7c387",
       "metadata": {
         "id": "inR70nh38PeK"
       },
@@ -710,6 +766,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "6eb614be",
       "metadata": {
         "id": "Nz0nasrh8T3c"
       },
@@ -720,6 +777,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "01448e2a",
       "metadata": {
         "id": "container:training,prediction,xgboost"
       },
@@ -738,6 +796,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "ece742dc",
       "metadata": {
         "id": "XujRA5ueox9U"
       },
@@ -750,6 +809,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "32a2397a",
       "metadata": {
         "id": "t1NLYz1R-KWv"
       },
@@ -759,6 +819,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "64570881",
       "metadata": {
         "id": "jnfKxpj0-Z0H"
       },
@@ -771,6 +832,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "f612dbf2",
       "metadata": {
         "id": "jv_-vU46_eFN"
       },
@@ -909,6 +971,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "bf048b2a",
       "metadata": {
         "id": "U1UiTZhkVoFM"
       },
@@ -921,6 +984,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "7684850a",
       "metadata": {
         "id": "9Gfr6pNLU-dB"
       },
@@ -943,6 +1007,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "cb6cae0b",
       "metadata": {
         "id": "RkfZ7qVAVjBO"
       },
@@ -953,6 +1018,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "c6b9ec3f",
       "metadata": {
         "id": "oYlLBGUSVibG"
       },
@@ -963,6 +1029,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "cc940f17",
       "metadata": {
         "id": "95vG4-zPWc0B"
       },
@@ -972,6 +1039,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "bb2b2eb4",
       "metadata": {
         "id": "ZNb6kZ2l5t-O"
       },
@@ -984,6 +1052,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "77314a6f",
       "metadata": {
         "id": "XPy0Jc8xXgpa"
       },
@@ -1001,6 +1070,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "aee97ebf",
       "metadata": {
         "id": "G0hm1no_WY8o"
       },
@@ -1024,6 +1094,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "0ca08588",
       "metadata": {
         "id": "O8TV4q535c2M"
       },
@@ -1036,6 +1107,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "a65f2574",
       "metadata": {
         "id": "dlCEJKfH5xR7"
       },
@@ -1047,6 +1119,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "f605666a-9f2a-479f-9948-6f2f29e4e76b",
       "metadata": {
         "id": "98c022ca36b4"
       },
@@ -1057,6 +1130,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "dc6661c5",
       "metadata": {
         "id": "FA9W85vs7LLD"
       },
@@ -1083,6 +1157,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "ef041ba2",
       "metadata": {
         "id": "ISsK9Msi-Kqs"
       },
@@ -1097,6 +1172,7 @@
     },
     {
       "cell_type": "markdown",
+      "id": "02a718ab",
       "metadata": {
         "id": "TpV-iwP9qw9c"
       },
@@ -1112,6 +1188,7 @@
     {
       "cell_type": "code",
       "execution_count": null,
+      "id": "e90fb0b1",
       "metadata": {
         "id": "6xbYQn5t5Noe"
       },

notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb

@@ -44,7 +44,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",

notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb

@@ -33,18 +33,18 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_classification_online_explain.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_classification_online_explain.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_online_explain.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",

notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb

@@ -44,7 +44,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",

notebooks/official/explainable_ai/sdk_custom_image_classification_online_explain.ipynb

@@ -33,18 +33,18 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_image_classification_online_explain.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_custom_image_classification_online_explain.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_image_classification_online_explain.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_custom_image_classification_online_explain.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/explainable_ai/sdk_custom_image_classification_online_explain.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official/explainable_ai/sdk_custom_image_classification_online_explain.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",

notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb

@@ -33,18 +33,18 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_custom_tabular_regression_batch_explain.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_custom_tabular_regression_batch_explain.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",

notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb

@@ -33,18 +33,18 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/tree/master/notebooks/official/automl/sdk_custom_tabular_regression_online_explain.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/notebooks/official/automl/sdk_custom_tabular_regression_online_explain.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",

notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb

@@ -33,20 +33,19 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/tree/master/notebooks/official/automl/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/notebooks/official/automl/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/notebooks/official/automl/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb\">\n",
+        "      Open in Google Cloud Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "</table>\n",

notebooks/official/feature_store/sdk-feature-store.ipynb

@@ -33,18 +33,16 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/sdk-feature-store.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/sdk-feature-store.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/sdk-feature-store.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/sdk-feature-store.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
-        "    </td>\n",
-        "    <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/sdk-feature-store.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>                                                                                           \n",
@@ -73,12 +71,14 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this notebook, you learn how to use `Vertex AI Feature Store` to import feature data, and to access the feature data for both online serving and offline tasks, such as training.\n",
+        "In this notebook, you will learn how to use `Vertex AI Feature Store` to import feature data, and to access the feature data for both online serving and offline tasks, such as training.\n",
         "\n",
         "This tutorial uses the following Google Cloud ML services:\n",
         "\n",
         "- `Vertex AI Feature Store`\n",
         "\n",
+        "\n",
+        "\n",
         "The steps performed include:\n",
         "\n",
         "- Create featurestore, entity type, and feature resources.\n",
@@ -393,7 +393,7 @@
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Vertex AI Workbench notebooks**, your environment is already\n",
-        "authenticated."
+        "authenticated. Skip this step."
       ]
     },
     {
@@ -822,7 +822,7 @@
       "source": [
         "## Import Feature Values\n",
         "\n",
-        "You need to import feature values before you can use them for online/offline serving. In this step, you learn how to import feature values by ingesting the values from Cloud Storage. You can also import feature values from BigQuery or a Pandas dataframe.\n"
+        "You need to import feature values before you can use them for online/offline serving. In this step, you learn how to import feature values by ingesting the values from GCS (Google Cloud Storage). You can also import feature values from BigQuery or a Pandas dataframe.\n"
       ]
     },
     {
@@ -1025,7 +1025,7 @@
       "source": [
         "### Read one entity per request\n",
         "\n",
-        "With the Vertex AI SDK, it is easy to read feature values of one entity. By default, the SDK will return the  latest value of each feature, meaning the feature values with the most recent  timestamp.\n",
+        "With the Python SDK, it is easy to read feature values of one entity. By default, the SDK will return the  latest value of each feature, meaning the feature values with the most recent  timestamp.\n",
         "\n",
         "To read feature values, specify the entity type ID and features to read. By default all the features of an entity type will be selected. The response will output and display the selected entity type ID and the selected feature values as a Pandas dataframe."
       ]

notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb

@@ -29,12 +29,12 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# Create Vertex AI Matching Engine index\n",
-        "\n",
         "<table align=\"left\">\n",
+        "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Run in Vertex Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
@@ -43,64 +43,38 @@
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
-        "      <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>\n",
         "</table>"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "b0a74aaf1481"
-      },
-      "source": [
-        "## Overview\n",
-        "\n",
-        "This example demonstrates how to use the GCP ANN Service. It is a high scale, low latency solution, to find similar vectors (or more specifically \"embeddings\") for a large corpus. Moreover, it is a fully managed offering, further reducing operational overhead. It is built upon [Approximate Nearest Neighbor (ANN) technology](https://ai.googleblog.com/2020/07/announcing-scann-efficient-vector.html) developed by Google Research."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "34a4b245e795"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "In this notebook, you learn how to create Approximate Nearest Neighbor (ANN) Index, query against indexes, and validate the performance of the index. \n",
-        "\n",
-        "The steps performed include:\n",
-        "\n",
-        "* Create ANN Index and Brute Force Index\n",
-        "* Create an IndexEndpoint with VPC Network\n",
-        "* Deploy ANN Index and Brute Force Index\n",
-        "* Perform online query\n",
-        "* Compute recall\n"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "tvgnzT1CKxrO"
       },
       "source": [
+        "## Overview\n",
+        "\n",
+        "This example demonstrates how to use the GCP ANN Service. It is a high scale, low latency solution, to find similar vectors (or more specifically \"embeddings\") for a large corpus. Moreover, it is a fully managed offering, further reducing operational overhead. It is built upon [Approximate Nearest Neighbor (ANN) technology](https://ai.googleblog.com/2020/07/announcing-scann-efficient-vector.html) developed by Google Research.\n",
+        "\n",
         "### Dataset\n",
         "\n",
         "The dataset used for this tutorial is the [GloVe dataset](https://nlp.stanford.edu/projects/glove/).\n",
         "\n",
-        "\"GloVe is an unsupervised learning algorithm for obtaining vector representations for words. Training is performed on aggregated global word-word co-occurrence statistics from a corpus, and the resulting representations showcase interesting linear substructures of the word vector space.\"\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5e2eba58ad71"
-      },
-      "source": [
+        "\"GloVe is an unsupervised learning algorithm for obtaining vector representations for words. Training is performed on aggregated global word-word co-occurrence statistics from a corpus, and the resulting representations showcase interesting linear substructures of the word vector space.\"\n",
+        "\n",
+        "### Objective\n",
+        "\n",
+        "In this notebook, you will learn how to create Approximate Nearest Neighbor (ANN) Index, query against indexes, and validate the performance of the index. \n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "* Create ANN Index and Brute Force Index\n",
+        "* Create an IndexEndpoint with VPC Network\n",
+        "* Deploy ANN Index and Brute Force Index\n",
+        "* Perform online query\n",
+        "* Compute recall\n",
+        "\n",
+        "\n",
         "### Costs \n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
@@ -115,14 +89,21 @@
         "to generate a cost estimate based on your projected usage."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "S5zc4kbEiYCm"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "d1e95a984673"
       },
       "source": [
-        "## Before you begin\n",
-        "\n",
         "### Set up your Google Cloud project\n",
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
@@ -187,7 +168,18 @@
       "metadata": {
         "id": "3e2b43c2d2bf"
       },
-      "outputs": [],
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Your browser has been opened to visit:\n",
+            "\n",
+            "    https://accounts.google.com/o/oauth2/auth?response_type=code&client_id=32555940559.apps.googleusercontent.com&redirect_uri=http%3A%2F%2Flocalhost%3A8085%2F&scope=openid+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fuserinfo.email+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcloud-platform+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fappengine.admin+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fsqlservice.login+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcompute+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Faccounts.reauth&state=UY9jjYfhoSedWWUOWXp5Pmicq0Ic04&access_type=offline&code_challenge=OQefcewSwkT7ZwfzzOVidtngvZspdY1NgN6rltw8x7A&code_challenge_method=S256\n",
+            "\n"
+          ]
+        }
+      ],
       "source": [
         "import os\n",
         "import sys\n",
@@ -227,11 +219,19 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 1,
       "metadata": {
         "id": "beb72f394541"
       },
-      "outputs": [],
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Project ID:  python-docs-samples-tests\n"
+          ]
+        }
+      ],
       "source": [
         "import os\n",
         "\n",

notebooks/official/migration/UJ11 Vertex SDK Hyperparameter Tuning.ipynb

@@ -32,24 +32,17 @@
         "# Vertex AI: Vertex AI Migration: Hyperparameter Tuning\n",
         "\n",
         "<table align=\"left\">\n",
-        "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ11 Vertex SDK Hyperparameter Tuning.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ11%20Vertex%20SDK%20Hyperparameter%20Tuning.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ11 Vertex SDK Hyperparameter Tuning.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ11%20Vertex%20SDK%20Hyperparameter%20Tuning.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ11 Vertex SDK Hyperparameter Tuning.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>                                                                                               \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -62,7 +55,7 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "The dataset used for this tutorial is the [Boston Housing Prices dataset](https://www.cs.toronto.edu/~delve/data/boston/bostonDetail.html). The version of the dataset you use in this tutorial is built into TensorFlow. The trained model predicts the median price of a house in units of 1K USD."
+        "The dataset used for this tutorial is the [Boston Housing Prices dataset](https://www.cs.toronto.edu/~delve/data/boston/bostonDetail.html). The version of the dataset you will use in this tutorial is built into TensorFlow. The trained model predicts the median price of a house in units of 1K USD."
       ]
     },
     {
@@ -145,7 +138,7 @@
         "else:\n",
         "    USER_FLAG = \"\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q"
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
       ]
     },
     {
@@ -165,7 +158,7 @@
       },
       "outputs": [],
       "source": [
-        "! pip3 install -U google-cloud-storage $USER_FLAG -q"
+        "! pip3 install -U google-cloud-storage $USER_FLAG"
       ]
     },
     {
@@ -219,7 +212,7 @@
         "\n",
         "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
         "\n",
-        "4. If you are running this notebook locally, you need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
+        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
         "\n",
         "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
@@ -292,10 +285,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -304,9 +294,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### UUID\n",
+        "#### Timestamp\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -317,16 +307,9 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
@@ -337,7 +320,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebooks**, your environment is already authenticated.\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -372,11 +355,8 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebook, then don't execute this code\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -412,8 +392,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -424,9 +403,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -446,7 +424,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -466,7 +444,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -511,7 +489,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -533,7 +511,7 @@
         "\n",
         "Learn more [here](https://cloud.google.com/vertex-ai/docs/general/locations#accelerators) hardware accelerator support for your region\n",
         "\n",
-        "*Note*: TF releases before 2.3 for GPU support  fail to load the custom model in this tutorial. It is a known issue and fixed in TF 2.3 -- which is caused by static graph ops that are generated in the serving function. If you encounter this issue on your own custom models, use a container image for TF 2.3 with GPU support."
+        "*Note*: TF releases before 2.3 for GPU support will fail to load the custom model in this tutorial. It is a known issue and fixed in TF 2.3 -- which is caused by static graph ops that are generated in the serving function. If you encounter this issue on your own custom models, use a container image for TF 2.3 with GPU support."
       ]
     },
     {
@@ -544,8 +522,6 @@
       },
       "outputs": [],
       "source": [
-        "import os\n",
-        "\n",
         "if os.getenv(\"IS_TESTING_TRAIN_GPU\"):\n",
         "    TRAIN_GPU, TRAIN_NGPU = (\n",
         "        aip.gapic.AcceleratorType.NVIDIA_TESLA_K80,\n",
@@ -629,7 +605,7 @@
         "\n",
         "Next, set the machine type to use for training and prediction.\n",
         "\n",
-        "- Set the variables `TRAIN_COMPUTE` and `DEPLOY_COMPUTE` to configure  the compute resources for the VMs you use for for training and prediction.\n",
+        "- Set the variables `TRAIN_COMPUTE` and `DEPLOY_COMPUTE` to configure  the compute resources for the VMs you will use for for training and prediction.\n",
         " - `machine type`\n",
         "     - `n1-standard`: 3.75GB of memory per vCPU.\n",
         "     - `n1-highmem`: 6.5GB of memory per vCPU\n",
@@ -681,7 +657,7 @@
         "\n",
         "#### Package layout\n",
         "\n",
-        "Before you start the training, you look at how a Python package is assembled for a custom training job. When unarchived, the package contains the following directory/file layout.\n",
+        "Before you start the training, you will look at how a Python package is assembled for a custom training job. When unarchived, the package contains the following directory/file layout.\n",
         "\n",
         "- PKG-INFO\n",
         "- README.md\n",
@@ -697,7 +673,7 @@
         "\n",
         "#### Package Assembly\n",
         "\n",
-        "In the following cells, you assemble the training package."
+        "In the following cells, you will assemble the training package."
       ]
     },
     {
@@ -745,7 +721,7 @@
         "- Build a DNN model.\n",
         "- The number of units per dense layer and learning rate hyperparameter values are used during the build and compile of the model.\n",
         "- A definition of a callback `HPTCallback` which obtains the validation loss at the end of each epoch (`on_epoch_end()`) and reports it to the hyperparameter tuning service using `hpt.report_hyperparameter_tuning_metric()`.\n",
-        "- Train the model with the `fit()` method and specify a callback which report the validation loss back to the hyperparameter tuning service."
+        "- Train the model with the `fit()` method and specify a callback which will report the validation loss back to the hyperparameter tuning service."
       ]
     },
     {
@@ -878,7 +854,7 @@
         "! rm -f custom.tar custom.tar.gz\n",
         "! tar cvf custom.tar custom\n",
         "! gzip custom.tar\n",
-        "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz"
+        "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz"
       ]
     },
     {
@@ -909,7 +885,7 @@
         "\n",
         "Now define the machine specification for your custom training job. This tells Vertex what type of machine instance to provision for the training.\n",
         "  - `machine_type`: The type of GCP instance to provision -- e.g., n1-standard-8.\n",
-        "  - `accelerator_type`: The type, if any, of hardware accelerator. In this tutorial if you previously set the variable `TRAIN_GPU != None`, you are using a GPU; otherwise you use a CPU.\n",
+        "  - `accelerator_type`: The type, if any, of hardware accelerator. In this tutorial if you previously set the variable `TRAIN_GPU != None`, you are using a GPU; otherwise you will use a CPU.\n",
         "  - `accelerator_count`: The number of accelerators."
       ]
     },
@@ -967,7 +943,7 @@
       "source": [
         "### Define the worker pool specification\n",
         "\n",
-        "Next, you define the worker pool specification for your custom training job. The worker pool specification  consist of the following:\n",
+        "Next, you define the worker pool specification for your custom training job. The worker pool specification will consist of the following:\n",
         "\n",
         "- `replica_count`: The number of instances to provision of this machine type.\n",
         "- `machine_spec`: The hardware specification.\n",
@@ -979,11 +955,11 @@
         "\n",
         "-`executor_image_spec`: This is the docker image which is configured for your custom training job.\n",
         "\n",
-        "-`package_uris`: This is a list of the locations (URIs) of your python training packages to install on the provisioned instance. The locations need to be in a Cloud Storage bucket. These can be either individual python files or a zip (archive) of an entire package. In the later case, the job service  unzip (unarchive) the contents into the docker image.\n",
+        "-`package_uris`: This is a list of the locations (URIs) of your python training packages to install on the provisioned instance. The locations need to be in a Cloud Storage bucket. These can be either individual python files or a zip (archive) of an entire package. In the later case, the job service will unzip (unarchive) the contents into the docker image.\n",
         "\n",
-        "-`python_module`: The Python module (script) to invoke for running the custom training job. In this example, you  be invoking `trainer.task.py` -- note that it was not neccessary to append the `.py` suffix.\n",
+        "-`python_module`: The Python module (script) to invoke for running the custom training job. In this example, you will be invoking `trainer.task.py` -- note that it was not neccessary to append the `.py` suffix.\n",
         "\n",
-        "-`args`: The command line arguments to pass to the corresponding Pythom module. In this example, you  be setting:\n",
+        "-`args`: The command line arguments to pass to the corresponding Pythom module. In this example, you will be setting:\n",
         "  - `\"--model-dir=\" + MODEL_DIR` : The Cloud Storage location where to store the model artifacts. There are two ways to tell the training script where to save the model artifacts:\n",
         "      - direct: You pass the Cloud Storage location as a command line argument to your training script (set variable `DIRECT = True`), or\n",
         "      - indirect: The service passes the Cloud Storage location as the environment variable `AIP_MODEL_DIR` to your training script (set variable `DIRECT = False`). In this case, you tell the service the model artifact location in the job specification.\n",
@@ -1003,8 +979,8 @@
       },
       "outputs": [],
       "source": [
-        "JOB_NAME = \"custom_job_\" + UUID\n",
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, JOB_NAME)\n",
+        "JOB_NAME = \"custom_job_\" + TIMESTAMP\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, JOB_NAME)\n",
         "\n",
         "if not TRAIN_NGPU or TRAIN_NGPU < 2:\n",
         "    TRAIN_STRATEGY = \"single\"\n",
@@ -1036,7 +1012,7 @@
         "        \"disk_spec\": disk_spec,\n",
         "        \"python_package_spec\": {\n",
         "            \"executor_image_uri\": TRAIN_IMAGE,\n",
-        "            \"package_uris\": [BUCKET_URI + \"/trainer_boston.tar.gz\"],\n",
+        "            \"package_uris\": [BUCKET_NAME + \"/trainer_boston.tar.gz\"],\n",
         "            \"python_module\": \"trainer.task\",\n",
         "            \"args\": CMDARGS,\n",
         "        },\n",
@@ -1075,7 +1051,9 @@
       },
       "outputs": [],
       "source": [
-        "job = aip.CustomJob(display_name=\"boston_\" + UUID, worker_pool_specs=worker_pool_spec)\n",
+        "job = aip.CustomJob(\n",
+        "    display_name=\"boston_\" + TIMESTAMP, worker_pool_specs=worker_pool_spec\n",
+        ")\n",
         "\n",
         "# print(job)"
       ]
@@ -1107,7 +1085,7 @@
         "from google.cloud.aiplatform import hyperparameter_tuning as hpt\n",
         "\n",
         "hpt_job = aip.HyperparameterTuningJob(\n",
-        "    display_name=\"boston_\" + UUID,\n",
+        "    display_name=\"boston_\" + TIMESTAMP,\n",
         "    custom_job=job,\n",
         "    metric_spec={\n",
         "        \"val_loss\": \"minimize\",\n",
@@ -1177,7 +1155,7 @@
       "source": [
         "### Display the hyperparameter tuning job trial results\n",
         "\n",
-        "After the hyperparameter tuning job has completed, the property `trials`  return the results for each trial."
+        "After the hyperparameter tuning job has completed, the property `trials` will return the results for each trial."
       ]
     },
     {
@@ -1372,7 +1350,7 @@
         "        print(e)\n",
         "\n",
         "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_URI"
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/official/migration/UJ15 Vertex SDK AutoML Object Tracking.ipynb

@@ -32,24 +32,17 @@
         "# Vertex AI: Vertex AI Migration: AutoML Video Object Tracking\n",
         "\n",
         "<table align=\"left\">\n",
-        "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ15 Vertex SDK AutoML Object Tracking.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/ai-platform-samples/blob/main/vertex-ai-samples/tree/master/notebooks/official/migration/UJ15%20Vertex%20SDK%20AutoML%20Object%20Tracking.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ15 Vertex SDK AutoML Object Tracking.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/blob/main/notebooks/official/migration/UJ15%20Vertex%20SDK%20AutoML%20Object%20Tracking.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ15 Vertex SDK AutoML Object Tracking.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>                                                                                               \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -62,7 +55,7 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "The dataset used for this tutorial is the Traffic. The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket."
+        "The dataset used for this tutorial is the Traffic. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket."
       ]
     },
     {
@@ -145,7 +138,39 @@
         "else:\n",
         "    USER_FLAG = \"\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform  google-cloud-storage tensorflow $USER_FLAG -q"
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "install_storage"
+      },
+      "source": [
+        "Install the latest GA version of *google-cloud-storage* library as well."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_storage"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install -U google-cloud-storage $USER_FLAG"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_tensorflow"
+      },
+      "outputs": [],
+      "source": [
+        "if os.getenv(\"IS_TESTING\"):\n",
+        "    ! pip3 install --upgrade tensorflow $USER_FLAG"
       ]
     },
     {
@@ -199,7 +224,7 @@
         "\n",
         "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
         "\n",
-        "4. If you are running this notebook locally, you need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
+        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
         "\n",
         "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
@@ -272,10 +297,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -284,9 +306,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### UUID\n",
+        "#### Timestamp\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -297,16 +319,9 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
@@ -317,7 +332,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated.\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -352,11 +367,8 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebook, then don't execute this code\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -366,7 +378,7 @@
         "    # path to your service account key and run this cell to authenticate your GCP\n",
         "    # account.\n",
         "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS '[your-service-account-key-path]'"
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
       ]
     },
     {
@@ -379,7 +391,7 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "When you initialize the Vertex AI SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
+        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
         "\n",
         "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
       ]
@@ -392,8 +404,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -404,9 +415,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -426,7 +436,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -446,7 +456,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -478,9 +488,9 @@
         "id": "init_aip:mbsdk"
       },
       "source": [
-        "## Initialize Vertex AI SDK for Python\n",
+        "## Initialize Vertex SDK for Python\n",
         "\n",
-        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
+        "Initialize the Vertex SDK for Python for your project and corresponding bucket."
       ]
     },
     {
@@ -491,7 +501,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -592,7 +602,7 @@
       "outputs": [],
       "source": [
         "dataset = aip.VideoDataset.create(\n",
-        "    display_name=\"Traffic\" + \"_\" + UUID,\n",
+        "    display_name=\"Traffic\" + \"_\" + TIMESTAMP,\n",
         "    gcs_source=[IMPORT_FILE],\n",
         "    import_schema_uri=aip.schema.dataset.ioformat.video.object_tracking,\n",
         ")\n",
@@ -667,7 +677,7 @@
       "outputs": [],
       "source": [
         "dag = aip.AutoMLVideoTrainingJob(\n",
-        "    display_name=\"traffic_\" + UUID,\n",
+        "    display_name=\"traffic_\" + TIMESTAMP,\n",
         "    prediction_type=\"object_tracking\",\n",
         ")\n",
         "\n",
@@ -702,7 +712,7 @@
         "\n",
         "The `run` method when completed returns the `Model` resource.\n",
         "\n",
-        "The execution of the training pipeline take upto 20 minutes."
+        "The execution of the training pipeline will take upto 20 minutes."
       ]
     },
     {
@@ -715,7 +725,7 @@
       "source": [
         "model = dag.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"traffic_\" + UUID,\n",
+        "    model_display_name=\"traffic_\" + TIMESTAMP,\n",
         "    training_fraction_split=0.8,\n",
         "    test_fraction_split=0.2,\n",
         ")"
@@ -790,7 +800,7 @@
       "outputs": [],
       "source": [
         "# Get model resource ID\n",
-        "models = aip.Model.list(filter=\"display_name=traffic_\" + UUID)\n",
+        "models = aip.Model.list(filter=\"display_name=traffic_\" + TIMESTAMP)\n",
         "\n",
         "# Get a reference to the Model Service client\n",
         "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
@@ -869,7 +879,7 @@
       "source": [
         "### Get test item(s)\n",
         "\n",
-        "Now do a batch prediction to your Vertex model. You use arbitrary examples out of the dataset as a test items. Don't be concerned that the examples were likely used in training the model -- we just want to demonstrate how to make a prediction."
+        "Now do a batch prediction to your Vertex model. You will use arbitrary examples out of the dataset as a test items. Don't be concerned that the examples were likely used in training the model -- we just want to demonstrate how to make a prediction."
       ]
     },
     {
@@ -908,7 +918,7 @@
       "source": [
         "### Make a batch input file\n",
         "\n",
-        "Now make a batch input file, which you store in your local Cloud Storage bucket. The batch input file can be either CSV or JSONL. You use JSONL in this tutorial. For JSONL file, you make one dictionary entry per line for each video. The dictionary contains the key/value pairs:\n",
+        "Now make a batch input file, which you store in your local Cloud Storage bucket. The batch input file can be either CSV or JSONL. You will use JSONL in this tutorial. For JSONL file, you make one dictionary entry per line for each video. The dictionary contains the key/value pairs:\n",
         "\n",
         "- `content`: The Cloud Storage path to the video.\n",
         "- `mimeType`: The content type. In our example, it is a `avi` file.\n",
@@ -928,7 +938,7 @@
         "\n",
         "import tensorflow as tf\n",
         "\n",
-        "gcs_input_uri = BUCKET_URI + \"/test.jsonl\"\n",
+        "gcs_input_uri = BUCKET_NAME + \"/test.jsonl\"\n",
         "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
         "    data = {\n",
         "        \"content\": test_item_1,\n",
@@ -962,7 +972,7 @@
         "- `job_display_name`: The human readable name for the batch prediction job.\n",
         "- `gcs_source`: A list of one or more batch request input files.\n",
         "- `gcs_destination_prefix`: The Cloud Storage location for storing the batch prediction resuls.\n",
-        "- `sync`: If set to True, the call block while waiting for the asynchronous batch job to complete."
+        "- `sync`: If set to True, the call will block while waiting for the asynchronous batch job to complete."
       ]
     },
     {
@@ -974,9 +984,9 @@
       "outputs": [],
       "source": [
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"traffic_\" + UUID,\n",
+        "    job_display_name=\"traffic_\" + TIMESTAMP,\n",
         "    gcs_source=gcs_input_uri,\n",
-        "    gcs_destination_prefix=BUCKET_URI,\n",
+        "    gcs_destination_prefix=BUCKET_NAME,\n",
         "    sync=False,\n",
         ")\n",
         "\n",
@@ -1205,7 +1215,7 @@
         "        print(e)\n",
         "\n",
         "    if \"BUCKET_NAME\" in globals():\n",
-        "        ! gsutil rm -r $BUCKET_URI"
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/official/migration/UJ4 Vertex SDK AutoML Tabular Binary Classification.ipynb

@@ -32,64 +32,19 @@
         "# Vertex AI: Vertex AI Migration: AutoML Tabular Binary Classification\n",
         "\n",
         "<table align=\"left\">\n",
-        "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ4 Vertex SDK AutoML Tabular Binary Classification.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ4%20Vertex%20SDK%20AutoML%20Tabular%20Binary%20Classification.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ4 Vertex SDK AutoML Tabular Binary Classification.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ4%20Vertex%20SDK%20AutoML%20Tabular%20Binary%20Classification.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/migration/UJ4 Vertex SDK AutoML Tabular Binary Classification.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>                                                                                               \n",
-        "</table>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "fb82f94bbbc7"
-      },
-      "source": [
-        "## Overview\n",
-        "\n",
-        "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK to create tabular binary classification models and do online prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "9f80bba45dd5"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "In this tutorial, you create an AutoML tabular binary classification model and deploy for online prediction from a Python script using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
-        "\n",
-        "This tutorial uses the following Google Cloud ML services and resources:\n",
-        "\n",
-        "- Vertex AI managed Datasets\n",
-        "- Vertex AI Training\n",
-        "- Vertex AI Endpoints\n",
-        "- Vertex AI prediction\n",
-        "\n",
-        "The steps performed include:\n",
-        "\n",
-        "- Create a Vertex `Dataset` resource.\n",
-        "- Train the model.\n",
-        "- View the model evaluation.\n",
-        "- Deploy the `Model` resource to a serving `Endpoint` resource.\n",
-        "- Make a prediction.\n",
-        "- Undeploy the `Model`"
+        "</table>\n",
+        "<br/><br/><br/>"
       ]
     },
     {
@@ -100,7 +55,7 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "The dataset used for this tutorial is the [Bank Marketing](https://pantheon.corp.google.com/storage/browser/_details/cloud-ml-tables-data/bank-marketing.csv) . This dataset does not require any feature engineering. The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket."
+        "The dataset used for this tutorial is the Bank Marketing. This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket."
       ]
     },
     {
@@ -131,38 +86,29 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step.\n",
+        "If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
         "\n",
-        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
-        "You need the following:\n",
+        "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
         "\n",
-        "* The Google Cloud SDK\n",
-        "* Git\n",
-        "* Python 3\n",
-        "* virtualenv\n",
-        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "- The Cloud Storage SDK\n",
+        "- Git\n",
+        "- Python 3\n",
+        "- virtualenv\n",
+        "- Jupyter notebook running in a virtual environment with Python 3\n",
         "\n",
-        "The Google Cloud guide to [Setting up a Python development\n",
-        "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
-        "installation guide](https://jupyter.org/install) provide detailed instructions\n",
-        "for meeting these requirements. The following steps provide a condensed set of\n",
-        "instructions:\n",
+        "The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:\n",
         "\n",
-        "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
+        "1. [Install and initialize the SDK](https://cloud.google.com/sdk/docs/).\n",
         "\n",
-        "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
+        "2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).\n",
         "\n",
-        "1. [Install\n",
-        "   virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
-        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3.  Activate the virtual environment.\n",
         "\n",
-        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
-        "command-line in a terminal shell.\n",
+        "4. To install Jupyter, run `pip3 install jupyter` on the command-line in a terminal shell.\n",
         "\n",
-        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "5. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
         "\n",
-        "1. Open this notebook in the Jupyter Notebook Dashboard."
+        "6. Open this notebook in the Jupyter Notebook Dashboard.\n"
       ]
     },
     {
@@ -173,7 +119,7 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the following packages required to execute this notebook. "
+        "Install the latest version of Vertex SDK for Python."
       ]
     },
     {
@@ -186,18 +132,33 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
-        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
-        "    \"/opt/deeplearning/metadata/env_version\"\n",
-        ")\n",
-        "\n",
-        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
-        "USER_FLAG = \"\"\n",
-        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "# Google Cloud Notebook\n",
+        "if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    USER_FLAG = \"--user\"\n",
+        "else:\n",
+        "    USER_FLAG = \"\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform fsspec gcsfs $USER_FLAG"
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "install_storage"
+      },
+      "source": [
+        "Install the latest GA version of *google-cloud-storage* and *tensorflow* libraries as well."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_storage"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install -U google-cloud-storage tensorflow $USER_FLAG"
       ]
     },
     {
@@ -208,7 +169,7 @@
       "source": [
         "### Restart the kernel\n",
         "\n",
-        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
       ]
     },
     {
@@ -219,7 +180,6 @@
       },
       "outputs": [],
       "source": [
-        "# Automatically restart kernel after installs\n",
         "import os\n",
         "\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
@@ -233,38 +193,31 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "c27795e4f4a1"
+        "id": "before_you_begin:nogpu"
       },
       "source": [
         "## Before you begin\n",
         "\n",
+        "### GPU runtime\n",
+        "\n",
+        "This tutorial does not require a GPU runtime.\n",
+        "\n",
         "### Set up your Google Cloud project\n",
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
         "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
         "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
         "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
+        "3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
         "\n",
-        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
         "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "5. Enter your project ID in the cell below. Then run the  cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
         "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "1460fd744366"
-      },
-      "source": [
-        "#### Set your project ID\n",
-        "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
       ]
     },
     {
@@ -313,7 +266,7 @@
         "#### Region\n",
         "\n",
         "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
         "\n",
         "- Americas: `us-central1`\n",
         "- Europe: `europe-west4`\n",
@@ -321,7 +274,7 @@
         "\n",
         "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
         "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
       ]
     },
     {
@@ -332,10 +285,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -344,9 +294,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### UUID\n",
+        "#### Timestamp\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -357,52 +307,36 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "32e1cd21a5d5"
+        "id": "gcp_authenticate"
       },
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. \n",
+        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
-        "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions\n",
-        "when prompted to authenticate your account via oAuth.\n",
+        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
         "**Otherwise**, follow these steps:\n",
         "\n",
-        "1. In the Cloud Console, go to the [**Create service account key**\n",
-        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
         "\n",
-        "2. Click **Create service account**.\n",
+        "**Click Create service account**.\n",
         "\n",
-        "3. In the **Service account name** field, enter a name, and\n",
-        "   click **Create**.\n",
+        "In the **Service account name** field, enter a name, and click **Create**.\n",
         "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
         "\n",
-        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
-        "local environment.\n",
+        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
         "\n",
-        "6. Enter the path to your service account key as the\n",
-        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
       ]
     },
     {
@@ -421,11 +355,8 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebook, then don't execute this code\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -448,7 +379,7 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "When you initialize the Vertex AI SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
+        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
         "\n",
         "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
       ]
@@ -461,8 +392,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -473,9 +403,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -495,7 +424,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -515,7 +444,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -524,6 +453,9 @@
         "id": "setup_vars"
       },
       "source": [
+        "### Set up variables\n",
+        "\n",
+        "Next, set up some variables used throughout the tutorial.\n",
         "### Import libraries and define constants"
       ]
     },
@@ -535,8 +467,7 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip\n",
-        "import pandas as pd"
+        "import google.cloud.aiplatform as aip"
       ]
     },
     {
@@ -545,9 +476,9 @@
         "id": "init_aip:mbsdk"
       },
       "source": [
-        "## Initialize Vertex AI SDK for Python\n",
+        "## Initialize Vertex SDK for Python\n",
         "\n",
-        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
+        "Initialize the Vertex SDK for Python for your project and corresponding bucket."
       ]
     },
     {
@@ -663,7 +594,7 @@
       "outputs": [],
       "source": [
         "dataset = aip.TabularDataset.create(\n",
-        "    display_name=\"Bank Marketing\" + \"_\" + UUID, gcs_source=[IMPORT_FILE]\n",
+        "    display_name=\"Bank Marketing\" + \"_\" + TIMESTAMP, gcs_source=[IMPORT_FILE]\n",
         ")\n",
         "\n",
         "print(dataset.resource_name)"
@@ -747,13 +678,13 @@
       },
       "outputs": [],
       "source": [
-        "job = aip.AutoMLTabularTrainingJob(\n",
-        "    display_name=\"bank_\" + UUID,\n",
+        "dag = aip.AutoMLTabularTrainingJob(\n",
+        "    display_name=\"bank_\" + TIMESTAMP,\n",
         "    optimization_prediction_type=\"classification\",\n",
         "    optimization_objective=\"minimize-log-loss\",\n",
         ")\n",
         "\n",
-        "print(job)"
+        "print(dag)"
       ]
     },
     {
@@ -799,9 +730,9 @@
       },
       "outputs": [],
       "source": [
-        "model = job.run(\n",
+        "model = dag.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"bank_\" + UUID,\n",
+        "    model_display_name=\"bank_\" + TIMESTAMP,\n",
         "    training_fraction_split=0.6,\n",
         "    validation_fraction_split=0.2,\n",
         "    test_fraction_split=0.2,\n",
@@ -877,7 +808,7 @@
       "outputs": [],
       "source": [
         "# Get model resource ID\n",
-        "models = aip.Model.list(filter=\"display_name=bank_\" + UUID)\n",
+        "models = aip.Model.list(filter=\"display_name=bank_\" + TIMESTAMP)\n",
         "\n",
         "# Get a reference to the Model Service client\n",
         "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
@@ -956,7 +887,7 @@
       "source": [
         "### Make test items\n",
         "\n",
-        "You use synthetic data as a test data items. Don't be concerned that we are using synthetic data -- we just want to demonstrate how to make a prediction."
+        "You will use synthetic data as a test data items. Don't be concerned that we are using synthetic data -- we just want to demonstrate how to make a prediction."
       ]
     },
     {
@@ -967,7 +898,7 @@
       "source": [
         "### Make the batch input file\n",
         "\n",
-        "Now make a batch input file, which you store in your local Cloud Storage bucket. Unlike image, video and text, the batch input file for tabular is only supported for CSV. For CSV file, you make:\n",
+        "Now make a batch input file, which you will store in your local Cloud Storage bucket. Unlike image, video and text, the batch input file for tabular is only supported for CSV. For CSV file, you make:\n",
         "\n",
         "- The first line is the heading with the feature (fields) heading names.\n",
         "- Each remaining line is a separate prediction request with the corresponding feature values.\n",
@@ -991,7 +922,7 @@
         "\n",
         "! cut -d, -f1-16 tmp.csv > batch.csv\n",
         "\n",
-        "gcs_input_uri = BUCKET_URI + \"/test.csv\"\n",
+        "gcs_input_uri = BUCKET_NAME + \"/test.csv\"\n",
         "\n",
         "! gsutil cp batch.csv $gcs_input_uri"
       ]
@@ -1023,9 +954,9 @@
       "outputs": [],
       "source": [
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"bank_\" + UUID,\n",
+        "    job_display_name=\"bank_\" + TIMESTAMP,\n",
         "    gcs_source=gcs_input_uri,\n",
-        "    gcs_destination_prefix=BUCKET_URI,\n",
+        "    gcs_destination_prefix=BUCKET_NAME,\n",
         "    instances_format=\"csv\",\n",
         "    predictions_format=\"csv\",\n",
         "    sync=False,\n",
@@ -1133,20 +1064,21 @@
       },
       "outputs": [],
       "source": [
+        "import tensorflow as tf\n",
+        "\n",
         "bp_iter_outputs = batch_predict_job.iter_outputs()\n",
         "\n",
         "prediction_results = list()\n",
         "for blob in bp_iter_outputs:\n",
-        "    if blob.name.split(\"/\")[-1].startswith(\"prediction.results\"):\n",
+        "    if blob.name.split(\"/\")[-1].startswith(\"prediction\"):\n",
         "        prediction_results.append(blob.name)\n",
         "\n",
+        "tags = list()\n",
         "for prediction_result in prediction_results:\n",
         "    gfile_name = f\"gs://{bp_iter_outputs.bucket.name}/{prediction_result}\"\n",
-        "    df = pd.read_csv(gfile_name)\n",
-        "    print(f\"File name: {gfile_name}\")\n",
-        "    print(\"Prediction: \\n\\n\\n\\n\")\n",
-        "    print(df)\n",
-        "    print(\"\\n\\n\\n\")"
+        "    with tf.io.gfile.GFile(name=gfile_name, mode=\"r\") as gfile:\n",
+        "        for line in gfile.readlines():\n",
+        "            print(line)"
       ]
     },
     {
@@ -1157,20 +1089,11 @@
       "source": [
         "*Example Output:*\n",
         "\n",
-        "    File name: gs://vertex-ai-devaip-5j22pmou/prediction-bank_5j22pmou-2022_08_24T01_05_46_028Z/prediction.results-00005-of-00008.csv\n",
-        "Prediction: \n",
+        "    Age,Job,MaritalStatus,Education,Default,Balance,Housing,Loan,Contact,Day,Month,Duration,Campaign,PDays,Previous,POutcome,Deposit_1_scores,Deposit_2_scores\n",
         "\n",
+        "    72,retired,married,secondary,no,5715,no,no,cellular,17,nov,1127,5,184,3,success,0.4721628427505493,0.5278371572494507\n",
         "\n",
-        "\n",
-        "\n",
-        "    Age          Job MaritalStatus  Education Default  Balance Housing Loan  \\\n",
-        "0    57  blue-collar       married  secondary      no      668      no   no   \n",
-        "\n",
-        "     Contact  Day Month  Duration  Campaign  PDays  Previous POutcome  \\\n",
-        "0  telephone   17   nov       508         4     -1         0  unknown   \n",
-        "\n",
-        "    Deposit_1_scores  Deposit_2_scores  \n",
-        "0          0.847498          0.152502  "
+        "    57,blue-collar,married,secondary,no,668,no,no,telephone,17,nov,508,4,-1,0,unknown,0.9005520343780518,0.09944798052310944"
       ]
     },
     {
@@ -1250,7 +1173,7 @@
       "source": [
         "### Make test item\n",
         "\n",
-        "You use synthetic data as a test data item. Don't be concerned that we are using synthetic data -- we just want to demonstrate how to make a prediction."
+        "You will use synthetic data as a test data item. Don't be concerned that we are using synthetic data -- we just want to demonstrate how to make a prediction."
       ]
     },
     {
@@ -1370,10 +1293,13 @@
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
         "- Dataset\n",
+        "- Pipeline\n",
         "- Model\n",
         "- Endpoint\n",
         "- AutoML Training Job\n",
         "- Batch Job\n",
+        "- Custom Job\n",
+        "- Hyperparameter Tuning Job\n",
         "- Cloud Storage Bucket"
       ]
     },
@@ -1385,24 +1311,60 @@
       },
       "outputs": [],
       "source": [
-        "# Delete the dataset using the Vertex dataset object\n",
-        "dataset.delete()\n",
+        "delete_all = True\n",
         "\n",
-        "# Delete the model using the Vertex model object\n",
-        "model.delete()\n",
+        "if delete_all:\n",
+        "    # Delete the dataset using the Vertex dataset object\n",
+        "    try:\n",
+        "        if \"dataset\" in globals():\n",
+        "            dataset.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
         "\n",
-        "# Delete the endpoint using the Vertex endpoint object\n",
-        "endpoint.delete()\n",
+        "    # Delete the model using the Vertex model object\n",
+        "    try:\n",
+        "        if \"model\" in globals():\n",
+        "            model.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
         "\n",
-        "# Delete the AutoML or Pipeline trainig job\n",
-        "job.delete()\n",
+        "    # Delete the endpoint using the Vertex endpoint object\n",
+        "    try:\n",
+        "        if \"endpoint\" in globals():\n",
+        "            endpoint.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
         "\n",
-        "# Delete the batch prediction job using the Vertex batch prediction object\n",
-        "batch_predict_job.delete()\n",
+        "    # Delete the AutoML or Pipeline trainig job\n",
+        "    try:\n",
+        "        if \"dag\" in globals():\n",
+        "            dag.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
         "\n",
-        "delete_bucket = False\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil -m rm -r $BUCKET_URI"
+        "    # Delete the custom trainig job\n",
+        "    try:\n",
+        "        if \"job\" in globals():\n",
+        "            job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
+        "    try:\n",
+        "        if \"batch_predict_job\" in globals():\n",
+        "            batch_predict_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
+        "    try:\n",
+        "        if \"hpt_job\" in globals():\n",
+        "            hpt_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    if \"BUCKET_NAME\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/official/migration/UJ6 Vertex SDK AutoML Text Classification.ipynb

@@ -29,7 +29,7 @@
         "id": "title:migration,new"
       },
       "source": [
-        "# Vertex AI Migration : AutoML Text Classification\n",
+        "# Vertex AI: Vertex AI Migration: AutoML Text Classification\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -43,12 +43,6 @@
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
-        "      <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ6%20Vertex%20SDK%20AutoML%20Text%20Classification.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>   \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -56,49 +50,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "cc58cf7bf111"
-      },
-      "source": [
-        "## Overview\n",
-        "\n",
-        "<a name=\"section-1\"></a>\n",
-        "\n",
-        "This notebook demonstrates how to create an AutoML Video Classification Model, with a Vertex AI video dataset, and how to serve the model for batch prediction. It requires you provide a bucket where the dataset will be stored.\n",
-        "\n",
-        "Note: you may incur charges for training, prediction, storage or usage of other GCP products in connection with testing this SDK.\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "8ea675435b19"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "The objective of this notebook is to build a AutoML Video Classification Model. The following steps have been followed:\n",
-        "This tutorial uses the following Google Cloud ML services :\n",
-        "\n",
-        "* Vertex AI Dataset resource\n",
-        "* AutoML Training\n",
-        "* Vertex AI Model resource\n",
-        "* Vertex AI Batch Prediction\n",
-        "\n",
-        "The steps performed include the following:\n",
-        "\n",
-        "* Set your task name, and GCS prefix\n",
-        "* Copy AutoML video demo train data for creating managed dataset\n",
-        "* Create a dataset on Vertex AI.\n",
-        "* Configure a training job\n",
-        "* Launch a training job and create a model on Vertex AI\n",
-        "* Copy AutoML Video Demo Prediction Data for creating batch prediction job\n",
-        "* Perform batch prediction job on the model"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dd81fd5c3454"
+        "id": "dataset:happydb,tcn"
       },
       "source": [
         "### Dataset\n",
@@ -134,7 +86,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Vertex AI Workbench Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
         "\n",
         "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
         "\n",
@@ -167,7 +119,7 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the latest version of Vertex AI SDK for Python and GA version of *google-cloud-storage* library as well."
+        "Install the latest version of Vertex SDK for Python."
       ]
     },
     {
@@ -180,15 +132,43 @@
       "source": [
         "import os\n",
         "\n",
-        "# Vertex AI Workbench Notebook\n",
+        "# Google Cloud Notebook\n",
         "if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    USER_FLAG = \"--user\"\n",
         "else:\n",
         "    USER_FLAG = \"\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
-        "! pip3 install -U google-cloud-storage $USER_FLAG -q\n",
-        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "install_storage"
+      },
+      "source": [
+        "Install the latest GA version of *google-cloud-storage* library as well."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_storage"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install -U google-cloud-storage $USER_FLAG"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_tensorflow"
+      },
+      "outputs": [],
+      "source": [
         "if os.getenv(\"IS_TESTING\"):\n",
         "    ! pip3 install --upgrade tensorflow $USER_FLAG"
       ]
@@ -201,7 +181,7 @@
       "source": [
         "### Restart the kernel\n",
         "\n",
-        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+        "Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
       ]
     },
     {
@@ -252,17 +232,6 @@
         "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5aee4379e8e5"
-      },
-      "source": [
-        "#### Set your project ID\n",
-        "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -328,10 +297,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -340,9 +306,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### UUID\n",
+        "#### Timestamp\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial.\n"
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -353,16 +319,9 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
@@ -373,7 +332,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -408,11 +367,8 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebook, then don't execute this code\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -435,7 +391,7 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "When you initialize the Vertex AI SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
+        "When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
         "\n",
         "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
       ]
@@ -448,7 +404,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_URI = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -459,8 +415,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + UUID"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -480,7 +436,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gsutil mb -l $REGION $BUCKET_NAME"
       ]
     },
     {
@@ -500,7 +456,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_URI"
+        "! gsutil ls -al $BUCKET_NAME"
       ]
     },
     {
@@ -532,9 +488,9 @@
         "id": "init_aip:mbsdk"
       },
       "source": [
-        "## Initialize Vertex AI SDK for Python\n",
+        "## Initialize Vertex SDK for Python\n",
         "\n",
-        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
+        "Initialize the Vertex SDK for Python for your project and corresponding bucket."
       ]
     },
     {
@@ -545,7 +501,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
       ]
     },
     {
@@ -603,6 +559,24 @@
         "! gsutil cat $FILE | head"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_a_dataset:migration"
+      },
+      "source": [
+        "## Create a dataset"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "datasets_create:migration,new,mbsdk"
+      },
+      "source": [
+        "### [datasets.create-dataset-api](https://cloud.google.com/vertex-ai/docs/datasets/create-dataset-api)"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -611,8 +585,6 @@
       "source": [
         "### Create the Dataset\n",
         "\n",
-        "### [datasets.create-dataset-api](https://cloud.google.com/vertex-ai/docs/datasets/create-dataset-api)\n",
-        "\n",
         "Next, create the `Dataset` resource using the `create` method for the `TextDataset` class, which takes the following parameters:\n",
         "\n",
         "- `display_name`: The human readable name for the `Dataset` resource.\n",
@@ -631,7 +603,7 @@
       "outputs": [],
       "source": [
         "dataset = aip.TextDataset.create(\n",
-        "    display_name=\"Happy Moments\" + \"_\" + UUID,\n",
+        "    display_name=\"Happy Moments\" + \"_\" + TIMESTAMP,\n",
         "    gcs_source=[IMPORT_FILE],\n",
         "    import_schema_uri=aip.schema.dataset.ioformat.text.single_label_classification,\n",
         ")\n",
@@ -658,16 +630,30 @@
         "    projects/759209241365/locations/us-central1/datasets/3704325042721521664"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "train_a_model:migration"
+      },
+      "source": [
+        "## Train a model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "trainingpipelines_create:migration,new,mbsdk"
+      },
+      "source": [
+        "### [training.automl-api](https://cloud.google.com/vertex-ai/docs/training/automl-api)"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "create_automl_pipeline:text,tcn"
       },
       "source": [
-        "## Train a model\n",
-        "\n",
-        "### [training.automl-api](https://cloud.google.com/vertex-ai/docs/training/automl-api)\n",
-        "\n",
         "### Create and run training pipeline\n",
         "\n",
         "To train an AutoML model, you perform two steps: 1) create a training pipeline, and 2) run the pipeline.\n",
@@ -696,7 +682,7 @@
       "outputs": [],
       "source": [
         "dag = aip.AutoMLTextTrainingJob(\n",
-        "    display_name=\"happydb_\" + UUID,\n",
+        "    display_name=\"happydb_\" + TIMESTAMP,\n",
         "    prediction_type=\"classification\",\n",
         "    multi_label=False,\n",
         ")\n",
@@ -733,7 +719,7 @@
         "\n",
         "The `run` method when completed returns the `Model` resource.\n",
         "\n",
-        "The execution of the training pipeline will take up a few hours."
+        "The execution of the training pipeline will take upto 20 minutes."
       ]
     },
     {
@@ -746,7 +732,7 @@
       "source": [
         "model = dag.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"happydb_\" + UUID,\n",
+        "    model_display_name=\"happydb_\" + TIMESTAMP,\n",
         "    training_fraction_split=0.8,\n",
         "    validation_fraction_split=0.1,\n",
         "    test_fraction_split=0.1,\n",
@@ -778,6 +764,24 @@
         "    INFO:google.cloud.aiplatform.training_jobs:Model available at projects/759209241365/locations/us-central1/models/6389525951797002240"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "evaluate_the_model:migration"
+      },
+      "source": [
+        "## Evaluate the model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "models_evaluations_list:migration,new"
+      },
+      "source": [
+        "### [projects.locations.models.evaluations.list](https://cloud.devsite.corp.google.com/ai-platform-unified/docs/reference/rest/v1beta1/projects.locations.models.evaluations/list)"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -787,9 +791,7 @@
         "## Review model evaluation scores\n",
         "After your model has finished training, you can review the evaluation scores for it.\n",
         "\n",
-        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project.\n",
-        "\n",
-        "### [projects.locations.models.evaluations.list](https://cloud.devsite.corp.google.com/ai-platform-unified/docs/reference/rest/v1beta1/projects.locations.models.evaluations/list)"
+        "First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
       ]
     },
     {
@@ -801,7 +803,7 @@
       "outputs": [],
       "source": [
         "# Get model resource ID\n",
-        "models = aip.Model.list(filter=\"display_name=happydb_\" + UUID)\n",
+        "models = aip.Model.list(filter=\"display_name=happydb_\" + TIMESTAMP)\n",
         "\n",
         "# Get a reference to the Model Service client\n",
         "client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
@@ -854,14 +856,21 @@
         "              }"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "make_batch_predictions:migration"
+      },
+      "source": [
+        "## Make batch predictions"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "batchpredictionjobs_create:migration,new,mbsdk"
       },
       "source": [
-        "## Make batch predictions\n",
-        "\n",
         "### [predictions.batch-prediction](https://cloud.google.com/vertex-ai/docs/predictions/batch-predictions)"
       ]
     },
@@ -873,7 +882,7 @@
       "source": [
         "### Get test item(s)\n",
         "\n",
-        "Now do a batch prediction to your Vertex model. You will use arbitrary examples out of the dataset as a test items. Don't be concerned that the examples were likely used in training the model -- you just want to demonstrate how to make a prediction."
+        "Now do a batch prediction to your Vertex model. You will use arbitrary examples out of the dataset as a test items. Don't be concerned that the examples were likely used in training the model -- we just want to demonstrate how to make a prediction."
       ]
     },
     {
@@ -923,18 +932,17 @@
       "outputs": [],
       "source": [
         "import json\n",
-        "import os\n",
         "\n",
         "import tensorflow as tf\n",
         "\n",
-        "gcs_test_item_1 = BUCKET_URI + \"/test1.txt\"\n",
+        "gcs_test_item_1 = BUCKET_NAME + \"/test1.txt\"\n",
         "with tf.io.gfile.GFile(gcs_test_item_1, \"w\") as f:\n",
         "    f.write(test_item_1 + \"\\n\")\n",
-        "gcs_test_item_2 = BUCKET_URI + \"/test2.txt\"\n",
+        "gcs_test_item_2 = BUCKET_NAME + \"/test2.txt\"\n",
         "with tf.io.gfile.GFile(gcs_test_item_2, \"w\") as f:\n",
         "    f.write(test_item_2 + \"\\n\")\n",
         "\n",
-        "gcs_input_uri = BUCKET_URI + \"/test.jsonl\"\n",
+        "gcs_input_uri = BUCKET_NAME + \"/test.jsonl\"\n",
         "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
         "    data = {\"content\": gcs_test_item_1, \"mime_type\": \"text/plain\"}\n",
         "    f.write(json.dumps(data) + \"\\n\")\n",
@@ -970,9 +978,9 @@
       "outputs": [],
       "source": [
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"happydb_\" + UUID,\n",
+        "    job_display_name=\"happydb_\" + TIMESTAMP,\n",
         "    gcs_source=gcs_input_uri,\n",
-        "    gcs_destination_prefix=BUCKET_URI,\n",
+        "    gcs_destination_prefix=BUCKET_NAME,\n",
         "    sync=False,\n",
         ")\n",
         "\n",
@@ -1177,7 +1185,6 @@
         "id": "endpoints_predict:migration,new,mbsdk"
       },
       "source": [
-        "## Online prediction of automl\n",
         "### [predictions.online-prediction-automl](https://cloud.google.com/vertex-ai/docs/predictions/online-predictions-automl)"
       ]
     },
@@ -1189,7 +1196,7 @@
       "source": [
         "### Get test item\n",
         "\n",
-        "You will use an arbitrary example out of the dataset as a test item. Don't be concerned that the example was likely used in training the model -- you just want to demonstrate how to make a prediction."
+        "You will use an arbitrary example out of the dataset as a test item. Don't be concerned that the example was likely used in training the model -- we just want to demonstrate how to make a prediction."
       ]
     },
     {
@@ -1304,6 +1311,7 @@
         "- Endpoint\n",
         "- AutoML Training Job\n",
         "- Batch Job\n",
+        "- Custom Job\n",
         "- Hyperparameter Tuning Job\n",
         "- Cloud Storage Bucket"
       ]
@@ -1316,22 +1324,60 @@
       },
       "outputs": [],
       "source": [
-        "# Delete the dataset using the Vertex dataset object\n",
-        "dataset.delete()\n",
+        "delete_all = True\n",
         "\n",
-        "# Delete the model using the Vertex model object\n",
-        "model.delete()\n",
+        "if delete_all:\n",
+        "    # Delete the dataset using the Vertex dataset object\n",
+        "    try:\n",
+        "        if \"dataset\" in globals():\n",
+        "            dataset.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
         "\n",
-        "# Delete the endpoint using the Vertex endpoint object\n",
-        "endpoint.delete()\n",
+        "    # Delete the model using the Vertex model object\n",
+        "    try:\n",
+        "        if \"model\" in globals():\n",
+        "            model.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
         "\n",
-        "# Delete the batch prediction job using the Vertex batch prediction object\n",
-        "batch_predict_job.delete()\n",
+        "    # Delete the endpoint using the Vertex endpoint object\n",
+        "    try:\n",
+        "        if \"endpoint\" in globals():\n",
+        "            endpoint.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
         "\n",
-        "# Delete the bucket\n",
-        "delete_bucket = False\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil -m rm -r $BUCKET_URI"
+        "    # Delete the AutoML or Pipeline trainig job\n",
+        "    try:\n",
+        "        if \"dag\" in globals():\n",
+        "            dag.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the custom trainig job\n",
+        "    try:\n",
+        "        if \"job\" in globals():\n",
+        "            job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
+        "    try:\n",
+        "        if \"batch_predict_job\" in globals():\n",
+        "            batch_predict_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
+        "    try:\n",
+        "        if \"hpt_job\" in globals():\n",
+        "            hpt_job.delete()\n",
+        "    except Exception as e:\n",
+        "        print(e)\n",
+        "\n",
+        "    if \"BUCKET_NAME\" in globals():\n",
+        "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }
   ],

notebooks/official/migration/UJ8 Vertex SDK AutoML Text Sentiment Analysis.ipynb

@@ -32,24 +32,17 @@
         "# Vertex AI: Vertex AI Migration: AutoML Text Sentiment Analysis\n",
         "\n",
         "<table align=\"left\">\n",
-        "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ8 Vertex SDK AutoML Text Sentiment Analysis.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ8%20Vertex%20SDK%20AutoML%20Text%20Sentiment%20Analysis.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ8 Vertex SDK AutoML Text Sentiment Analysis.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ8%20Vertex%20SDK%20AutoML%20Text%20Sentiment%20Analysis.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ8 Vertex SDK AutoML Text Sentiment Analysis.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>                                                                                               \n",
         "</table>\n",
         "<br/><br/><br/>"
       ]
@@ -176,7 +169,8 @@
       },
       "outputs": [],
       "source": [
-        "! pip3 install --upgrade tensorflow $USER_FLAG"
+        "if os.getenv(\"IS_TESTING\"):\n",
+        "    ! pip3 install --upgrade tensorflow $USER_FLAG"
       ]
     },
     {

notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2021 Google LLC\n",
+        "# Copyright 2022 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -45,7 +45,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -56,39 +56,39 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "2e0464050974"
+        "id": "tvgnzT1CKxrO"
       },
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook demonstrates how to track metrics and parameters for Vertex AI custom training jobs, and how to perform detailed analysis using this data."
+        "This notebook demonstrates how to track metrics and parameters for `Vertex AI` custom training jobs, and how to perform detailed analysis using this data."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "b95ab729fccd"
+        "id": "37147bd9c3c4"
       },
       "source": [
         "### Objective\n",
         "\n",
-        "In this notebook, you learn how to use Vertex AI SDK for Python to:\n",
+        "In this notebook, you learn how to use `Vertex ML Metadata` to track training parameters and evaluation metrics.\n",
         "\n",
-        "This tutorial uses the following Google Cloud ML services and resources:\n",
-        "- Vertex AI Dataset\n",
-        "- Vertex AI Model\n",
-        "- Vertex AI Endpoint\n",
-        "- Vertex AI Custom Training Job\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex ML Metadata`\n",
+        "- `Vertex AI Experiments`\n",
         "\n",
         "The steps performed include:\n",
-        "- Track training parameters and prediction metrics for a custom training job.\n",
+        "\n",
+        "- Track parameters and metrics for a `Vertex AI` custom trained model.\n",
         "- Extract and perform analysis for all parameters and metrics within an Experiment."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "9fd87cf689bf"
+        "id": "96cb18467417"
       },
       "source": [
         "### Dataset\n",
@@ -99,7 +99,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "tvgnzT1CKxrO"
+        "id": "c831245dc1d5"
       },
       "source": [
         "### Costs \n",
@@ -181,14 +181,14 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "qblyW_dcyOQA"
+        "id": "IaYsrh0Tc17L"
       },
       "outputs": [],
       "source": [
         "import os\n",
         "\n",
         "# The Vertex AI Workbench Notebook product has specific requirements\n",
-        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
         "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
         "    \"/opt/deeplearning/metadata/env_version\"\n",
         ")\n",
@@ -198,10 +198,9 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "\n",
-        "! pip3 install -U tensorflow $USER_FLAG\n",
-        "! python3 -m pip3 install {USER_FLAG} google-cloud-aiplatform --upgrade\n",
-        "! pip3 install scikit-learn {USER_FLAG}"
+        "! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
+        "! pip3 install -U tensorflow $USER_FLAG -q\n",
+        "! pip3 install scikit-learn {USER_FLAG} -q"
       ]
     },
     {
@@ -286,7 +285,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "cde8e0876d62"
+        "id": "oM1iC_MfAts1"
       },
       "outputs": [],
       "source": [
@@ -297,11 +296,11 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "oM1iC_MfAts1"
+        "id": "riG_qUokg0XZ"
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "if PROJECT_ID == \"[your-project-id]\" or PROJECT_ID == \"\" or PROJECT_ID is None:\n",
         "    # Get your GCP project id from gcloud\n",
         "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
@@ -322,7 +321,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "47bc07d4231b"
+        "id": "region"
       },
       "source": [
         "#### Region\n",
@@ -336,14 +335,14 @@
         "\n",
         "You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
         "\n",
-        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+        "Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "959545da671a"
+        "id": "region"
       },
       "outputs": [],
       "source": [
@@ -359,9 +358,9 @@
         "id": "06571eb4063b"
       },
       "source": [
-        "#### UUID\n",
+        "#### Timestamp\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -372,16 +371,9 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of length 8\n",
-        "def generate_uuid():\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=8))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
@@ -393,7 +385,7 @@
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Vertex AI Workbench**, your environment is already\n",
-        "authenticated. Skip this step."
+        "authenticated. "
       ]
     },
     {
@@ -443,6 +435,7 @@
         "# requests.\n",
         "\n",
         "# If on Google Cloud Notebooks, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
         "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
@@ -467,7 +460,7 @@
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
         "\n",
-        "When you submit a training job using the Vertex AI SDK, you upload a Python package\n",
+        "When you submit a training job using the Cloud SDK, you upload a Python package\n",
         "containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
         "the code from this package. In this tutorial, Vertex AI also saves the\n",
         "trained model that results from your job in the same bucket. Using this model artifact, you can then\n",
@@ -499,8 +492,8 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
-        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
     {
@@ -622,7 +615,7 @@
       "outputs": [],
       "source": [
         "if EXPERIMENT_NAME == \"\" or EXPERIMENT_NAME is None:\n",
-        "    EXPERIMENT_NAME = \"my-experiment-\" + UUID"
+        "    EXPERIMENT_NAME = \"my-experiment-\" + TIMESTAMP"
       ]
     },
     {
@@ -662,10 +655,10 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "f8fd397cc4f6"
+        "id": "9nokDKBAxwV8"
       },
       "source": [
-        "### Download the Dataset to Cloud Storage"
+        "This example uses the Abalone Dataset. For more information about this dataset please visit: https://archive.ics.uci.edu/ml/datasets/abalone"
       ]
     },
     {
@@ -688,9 +681,9 @@
         "id": "35QVNhACqcTJ"
       },
       "source": [
-        "### Create a Vertex AI Tabular dataset from CSV data\n",
+        "### Create a Vertex AI Dataset from a CSV\n",
         "\n",
-        "A Vertex AI dataset can be used to create an AutoML model or a custom model. "
+        "A Vertex AI Dataset can be used to create an AutoML model or a custom model. "
       ]
     },
     {
@@ -703,7 +696,7 @@
       "source": [
         "ds = aiplatform.TabularDataset.create(display_name=\"abalone\", gcs_source=[gcs_csv_path])\n",
         "\n",
-        "ds.resource_name"
+        "print(ds.resource_name)"
       ]
     },
     {
@@ -714,7 +707,7 @@
       "source": [
         "### Write the training script\n",
         "\n",
-        "Next, you create the training script that is used in the sample custom training job."
+        "Run the following cell to create the training script that is used in the sample custom training job."
       ]
     },
     {
@@ -742,6 +735,9 @@
         "                    default=64, type=int,\n",
         "                    help='Number of unit for first layer.')\n",
         "args = parser.parse_args()\n",
+        "# uncomment and bump up replica_count for distributed training\n",
+        "# strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy()\n",
+        "# tf.distribute.experimental_set_strategy(strategy)\n",
         "\n",
         "col_names = [\"Length\", \"Diameter\", \"Height\", \"Whole weight\", \"Shucked weight\", \"Viscera weight\", \"Shell weight\", \"Age\"]\n",
         "target = \"Age\"\n",
@@ -775,7 +771,7 @@
         "id": "Yp2clkOJSDhR"
       },
       "source": [
-        "### Launch a custom training job and track its trainig parameters on Vertex ML Metadata"
+        "### Launch a custom training job and track its trainig parameters on Vertex AI ML Metadata"
       ]
     },
     {
@@ -801,7 +797,11 @@
         "id": "k_QorXXztzPH"
       },
       "source": [
-        "Start a new experiment run to track training parameters and start the training job. Note that this operation will take around 10 mins."
+        "Start a new experiment run to track training parameters and start the training job. \n",
+        "\n",
+        "Prior to executing the training job, you call the `start_run()` method to initialize the start of the experiment, and then use the `log_params()` to log the parameters used in the experiment.\n",
+        "\n",
+        "*Note:* This operation will take around 10 mins."
       ]
     },
     {
@@ -830,7 +830,7 @@
         "id": "5vhDsMJNqcTW"
       },
       "source": [
-        "### Deploy model and calculate prediction metrics"
+        "### Deploy Model and calculate prediction metrics"
       ]
     },
     {
@@ -839,7 +839,7 @@
         "id": "O-uCOL3Naap4"
       },
       "source": [
-        "Next, deploy your Vertex AI Model resource to a Vertex AI Endpoint resource. This operation will take 10-20 mins."
+        "Deploy model to Google Cloud. This operation may take a few minutes."
       ]
     },
     {
@@ -859,7 +859,7 @@
         "id": "JY-5skFhasWs"
       },
       "source": [
-        "### Prediction dataset preparation and online prediction"
+        "Once model is deployed, perform online prediction using the `abalone_test` dataset and calculate prediction metrics."
       ]
     },
     {
@@ -868,8 +868,6 @@
         "id": "saw50bqwa-dR"
       },
       "source": [
-        "Once model is deployed, perform online prediction using the `abalone_test` dataset and calculate prediction metrics.\n",
-        "\n",
         "Prepare the prediction dataset."
       ]
     },
@@ -922,7 +920,7 @@
         "id": "_HphZ38obJeB"
       },
       "source": [
-        "Perform online prediction."
+        "### Perform online prediction"
       ]
     },
     {
@@ -934,7 +932,7 @@
       "outputs": [],
       "source": [
         "prediction = endpoint.predict(test_dataset.tolist())\n",
-        "prediction"
+        "print(prediction)"
       ]
     },
     {
@@ -943,7 +941,11 @@
         "id": "TDKiv_O7bNwE"
       },
       "source": [
-        "Calculate and track prediction evaluation metrics."
+        "### Calculate and track prediction evaluation metrics.\n",
+        "\n",
+        "Next, log the evaluation metrics for your experiment.\n",
+        "\n",
+        "Once the experiment is completed, you call the `end_run()` method to indicate the end of tracking for the experiment."
       ]
     },
     {
@@ -957,7 +959,9 @@
         "mse = mean_squared_error(test_labels, prediction.predictions)\n",
         "mae = mean_absolute_error(test_labels, prediction.predictions)\n",
         "\n",
-        "aiplatform.log_metrics({\"mse\": mse, \"mae\": mae})"
+        "aiplatform.log_metrics({\"mse\": mse, \"mae\": mae})\n",
+        "\n",
+        "aiplatform.end_run()"
       ]
     },
     {

notebooks/official/model-registry/bqml-vertexai-model-registry.ipynb

@@ -29,23 +29,21 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# Deploy BiqQuery ML Model on Vertex AI Model Registry and make predictions\n",
-        "\n",
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model-registry/bqml-vertexai-model-registry.ipynb\" target=\"_blank\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/bigquery-ml/bqml-vertexai-model-registry.ipynb\" target=\"_blank\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model-registry/bqml-vertexai-model-registry.ipynb\" target=\"_blank\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/bigquery-ml/bqml-vertexai-model-registry.ipynb\" target=\"_blank\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/model-registry/bqml-vertexai-model-registry.ipynb\" target=\"_blank\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/bigquery-ml/bqml-vertexai-model-registry.ipynb\" target=\"_blank\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -59,9 +57,11 @@
         "id": "tvgnzT1CKxrO"
       },
       "source": [
+        "# Deploy BiqQuery ML Model on Vertex AI Model Registry and Make Predictions\n",
+        "\n",
         "## Overview\n",
         "\n",
-        "This tutorial demonstrates how to train a model with BigQuery ML and upload it on Vertex AI Model Registry, then make batch predictions.\n"
+        "This tutorial demonstrates how to train a model with BigQuery ML and upload it on Vertex AI model registry, then make batch predictions.\n"
       ]
     },
     {
@@ -77,13 +77,15 @@
         "This tutorial uses the following Google Cloud ML services and resources:\n",
         "\n",
         "- `Vertex AI Model Registry`\n",
+        "- `Vertex AI Model` resources \n",
         "- `Vertex AI Endpoint` resources\n",
+        "- `Vertex AI Prediction`\n",
         "- `BigQuery ML`\n",
         "\n",
         "\n",
         "The steps performed include:\n",
         "\n",
-        "- Train a model with `BigQuery ML`\n",
+        "- Train a model with `BQML`\n",
         "- Upload the model to `Vertex AI Model Registry` \n",
         "- Create a `Vertex AI Endpoint` resource\n",
         "- Deploy the `Model` resource to the `Endpoint` resource\n",
@@ -130,7 +132,15 @@
         "### Set up your local development environment\n",
         "\n",
         "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step.\n",
+        "all the requirements to run this notebook. You can skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gCuSR8GkAgzl"
+      },
+      "source": [
         "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
         "You need the following:\n",
         "\n",
@@ -175,7 +185,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 1,
+      "execution_count": null,
       "metadata": {
         "id": "2b4ef9b72d43"
       },
@@ -194,7 +204,8 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform google-cloud-bigquery pyarrow {USER_FLAG} -q"
+        "! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
+        "! pip3 install --upgrade google-cloud-bigquery {USER_FLAG} -q"
       ]
     },
     {
@@ -382,7 +393,15 @@
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated.\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sBCra4QMA2wR"
+      },
+      "source": [
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
         "\n",
@@ -524,7 +543,15 @@
         "id": "PKQD2e0eMg3M"
       },
       "source": [
-        "### Create BigQuery dataset resource\n",
+        "### Create BigQuery dataset resource"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BnXOpvs2MmzF"
+      },
+      "source": [
         "First, you create an empty dataset resource in your project."
       ]
     },
@@ -539,7 +566,17 @@
         "BQ_DATASET_NAME = \"penguins\" + UUID\n",
         "DATASET_QUERY = f\"\"\"CREATE SCHEMA {BQ_DATASET_NAME}\"\"\"\n",
         "\n",
-        "job = bqclient.query(DATASET_QUERY)\n",
+        "job = bqclient.query(DATASET_QUERY)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "59bf85366baf"
+      },
+      "outputs": [],
+      "source": [
         "job.result()\n",
         "print(job.state)"
       ]
@@ -551,7 +588,7 @@
       },
       "source": [
         "## Train BigQuery ML model and upload it to Vertex AI Model Registry\n",
-        "Next, you create and train a `BigQuery  ML` tabular regression model from the public dataset penguins and store the model in your project `Vertex AI Model Registry` using the `CREATE MODEL` statement. The model configuration is specified in the `OPTIONS` statement as follows:\n",
+        "Next, you create and train a BQML tabular regression model from the public dataset penguins and store the model in your project `Vertex AI Model Registry` using the `CREATE MODEL` statement. The model configuration is specified in the `OPTIONS` statement as follows:\n",
         "\n",
         "- `model_type`: The type and archictecture of tabular model to train, e.g., LOGISTIC_REG.\n",
         "\n",
@@ -594,7 +631,6 @@
         "id": "eee158e2a375"
       },
       "source": [
-        "### Create BigQuery ML Model\n",
         "Create the BigQuery ML model using the query above and the BigQuery client that you created previously:"
       ]
     },
@@ -641,7 +677,7 @@
       "source": [
         "### Find the model in the Vertex Model Registry\n",
         "\n",
-        "You can use the `Vertex AI Model()` method with `model_name` parameter to find the automatically registered model."
+        "You can use the `Vertex AI Model list()` method with a filter query to find the automatically registered model."
       ]
     },
     {
@@ -769,7 +805,15 @@
         "id": "C39qOaBHZI1G"
       },
       "source": [
-        "## Batch Prediction on the BigQuery ML model\n",
+        "## Batch Prediction on the BQML model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UBffk3GyaPY3"
+      },
+      "source": [
         "Here you request batch predictions directly from the BigQuery ML model; you don't need to deploy the model to an endpoint. For data types that support both batch and online predictions, use batch predictions when you don't require an immediate response and want to process accumulated data by using a single request.\n",
         "\n",
         "Learn more abount <a href=\"https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict\" target=\"_blank\">The ML.PREDICT function</a>"