chore: revisions to Stable Diffusion and TorchServe nb

Update wording for pre-built pytorch images on Vertex Prediction.

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -103,7 +103,6 @@ class EndpointResourceCleanupManager(VertexAIResourceCleanupManager):
             models.id for models in resource._gca_resource.deployed_models
         ]:
             resource._undeploy(deployed_model_id=deployed_model_id)
-
         resource.delete(force=True)
 
 
@@ -117,3 +116,7 @@ class MatchingEngineIndexResourceCleanupManager(VertexAIResourceCleanupManager):
 
 class MatchingEngineIndexEndpointResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.MatchingEngineIndexEndpoint
+
+    def delete(self, resource):
+        resource.undeploy_all()
+        resource.delete(force=True)

.cloud-build/requirements.txt

@@ -10,4 +10,4 @@ google-cloud-aiplatform
 google-cloud-storage
 google-cloud-build
 ratemate
-GitPython
+GitPython

.github/workflows/linter/requirements.txt

@@ -5,6 +5,6 @@ nbconvert
 black==22.10.0
 pyupgrade==2.38.4
 isort==5.10.1
-flake8==4.0.1
+flake8==6.0.0
 nbqa==1.5.3
 

community-content/pipeline_components/image_ml_model_training/pipeline.py

@@ -0,0 +1,62 @@
+# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
+from kfp import components
+from kfp.v2 import dsl
+
+# %% Loading components
+upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml')
+deploy_model_to_endpoint_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml')
+transcode_imagedataset_tfrecord_from_csv_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/community-content/pipeline_components/image_ml_model_training/transcode_tfrecord_image_dataset_from_csv/component.yaml')
+load_image_classification_model_from_tfhub_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/b5b65198a6c2ffe8c0fa2aa70127e3325752df68/community-content/pipeline_components/image_ml_model_training/load_image_classification_model/component.yaml')
+preprocess_image_data_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/community-content/pipeline_components/image_ml_model_training/preprocess_image_data/component.yaml')
+train_tensorflow_image_classification_model_op = components.load_component_from_url('https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/community-content/pipeline_components/image_ml_model_training/train_image_classification_model/component.yaml')
+
+
+# %% Pipeline definition
+def image_classification_pipeline():
+    class_names = ['daisy', 'dandelion', 'roses', 'sunflowers', 'tulips']
+    csv_image_data_path = 'gs://cloud-samples-data/ai-platform/flowers/flowers.csv'
+    deploy_model = False
+
+    image_data = dsl.importer(
+        artifact_uri=csv_image_data_path, artifact_class=dsl.Dataset).output
+
+    image_tfrecord_data = transcode_imagedataset_tfrecord_from_csv_op(
+        csv_image_data_path=image_data,
+        class_names=class_names
+    ).outputs['tfrecord_image_data_path']
+
+    loaded_model_outputs = load_image_classification_model_from_tfhub_op(
+        class_names=class_names,
+    ).outputs
+
+    preprocessed_data = preprocess_image_data_op(
+        image_tfrecord_data,
+        height_width_path=loaded_model_outputs['image_size_path'],
+    ).outputs
+
+    trained_model = (train_tensorflow_image_classification_model_op(
+        preprocessed_training_data_path = preprocessed_data['preprocessed_training_data_path'],
+        preprocessed_validation_data_path = preprocessed_data['preprocessed_validation_data_path'],
+        model_path=loaded_model_outputs['loaded_model_path']).
+                   set_cpu_limit('96').
+                   set_memory_limit('128G').
+                   add_node_selector_constraint('cloud.google.com/gke-accelerator', 'NVIDIA_TESLA_A100').
+                   set_gpu_limit('8').
+                   outputs['trained_model_path'])
+
+    vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
+        model=trained_model,
+    ).outputs['model_name']
+
+    # Deploying the model might incur additional costs over time
+    if deploy_model:
+        vertex_endpoint_name = deploy_model_to_endpoint_op(
+            model_name=vertex_model_name,
+        ).outputs['endpoint_name']
+
+pipeline_func = image_classification_pipeline
+
+# %% Pipeline submission
+if __name__ == '__main__':
+    from google.cloud import aiplatform
+    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

community-content/pytorch_pre_built_images_deployment/pytorch-text-classification-vertex-ai-deploy.ipynb

@@ -31,17 +31,7 @@
       "source": [
         "# Deploying a PyTorch Text Classification Model on [Vertex AI](https://cloud.google.com/vertex-ai)\n",
         "\n",
-        "**This is an Experimental release**, covered by the Pre-GA Offerings Terms of your Google Cloud Platform [Terms of Service](https://cloud.google.com/terms).\n",
-        "\n",
-        "Experiments are focused on validating a prototype and are not guaranteed to be released. They are not intended for production use or covered by any SLA, support obligation, or deprecation policy and might be subject to backward-incompatible changes.\n",
-        "\n",
-        "**Kindly drop us a note before you run any scale tests.**\n",
-        "\n",
-        "**Do not hesitate to contact vertexai-prediction-preview-feedback@google.com if you have any questions or run into any issues.**\n",
-        "\n",
-        "The usage of the product is free during the Experimental release period: you will still incur charges for other GCP products usage, such as storage.\n",
-        "\n",
-        "The projects need to be added to the allowlist in order to deploy PyTorch models using Vertex AI Prediction pre-built PyTorch images. If you are interested in the feature, please send an email to vertexai-prediction-preview-feedback@google.com to provide your project numbers OR project ids."
+        "**Kindly reach out to Vertex AI before you run any scale tests or you have any questions.**\n"
       ]
     },
     {

notebooks/community/CODEOWNERS

@@ -30,6 +30,7 @@
 /notebooks/community/neo4j/graph_paysim.ipynb @benofben @laeg
 /notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb @mansari
 /notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb @inardini
+/notebooks/community/cohere/cohere_embedding_with_matching_engine.ipynb @stewart-co
 /notebooks/community/ml_ops/stage2/get_started_vertex_hpt_r_kernel.ipynb @fhirschmann
 /notebooks/community/ml_ops/stage2/get_started_vertex_training_r_using_r_kernel.ipynb @fhirschmann
 /notebooks/community/vertex-ai-samples/notebooks/community/model_registry/vertex_ai_model_registry_bqml_custom_model_versioning.ipynb @inardini
@@ -37,3 +38,6 @@
 /notebooks/community/vizier/conversions_vertex_vizier_and_open_source_vizier.ipynb @halio-g
 /notebooks/community/experiments/vertex_ai_model_experimentation.ipynb @inardini @asobran
 /notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_anomaly_detection.ipynb @inardini
+/notebooks/community/pipelines/google_cloud_pipeline_components_cloud_natural_language_pipeline.ipynb @Narwhalprime
+/notebooks/community/pipelines/google_cloud_pipeline_components_ready_to_go_text_classification_pipeline.ipynb @Narwhalprime
+/notebooks/community/feature_store/get_started_vertex_feature_store.ipynb @junkourata

notebooks/community/cohere/README.md

@@ -0,0 +1,3 @@
+# README
+
+These are notebooks [Cohere](https://cohere.ai/) built in collaboration with Google.  They demonstrate how to use Cohere's modeling API along with Vertex AI.

notebooks/community/matching_engine/matching_engine_for_indexing.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2021 Google LLC\n",
+        "# Copyright 2023 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -24,6 +24,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "JAPoU8Sm5E6e"
@@ -32,20 +33,28 @@
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/matching_engine/matching_engine_for_indexing.ipynb\">\n",
-        "      Run in Google Cloud Notebooks\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/matching_engine/matching_engine_for_indexing.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\">\n",
+        "      Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/matching_engine/matching_engine_for_indexing.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/matching_engine/matching_engine_for_indexing.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/matching_engine/matching_engine_for_indexing.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>                                                                                               \n",
         "</table>"
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "tvgnzT1CKxrO"
@@ -53,25 +62,49 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This example demonstrates how to use the GCP ANN Service. It is a high scale, low latency solution, to find similar vectors (or more specifically \"embeddings\") for a large corpus. Moreover, it is a fully managed offering, further reducing operational overhead. It is built upon [Approximate Nearest Neighbor (ANN) technology](https://ai.googleblog.com/2020/07/announcing-scann-efficient-vector.html) developed by Google Research.\n",
-        "\n",
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [GloVe dataset](https://nlp.stanford.edu/projects/glove/).\n",
-        "\n",
+        "This example demonstrates how to use Vertex AI Matching Engine. It is a high scale, low latency solution, to find similar vectors (or more specifically \"embeddings\") for a large corpus. Moreover, it is a fully managed offering, further reducing operational overhead. It is built upon [Approximate Nearest Neighbor (ANN) technology](https://ai.googleblog.com/2020/07/announcing-scann-efficient-vector.html) developed by Google Research."
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "56e5f9699c6c"
+      },
+      "source": [
         "### Objective\n",
         "\n",
         "In this notebook, you will learn how to create Approximate Nearest Neighbor (ANN) Index, query against indexes, and validate the performance of the index. \n",
         "\n",
         "The steps performed include:\n",
         "\n",
-        "* Create ANN Index and Brute Force Index\n",
+        "* Create a Vertex AI Matching Engine Index and Brute Force Index\n",
         "* Create an IndexEndpoint with VPC Network\n",
-        "* Deploy ANN Index and Brute Force Index\n",
-        "* Perform online query\n",
-        "* Compute recall\n",
-        "\n",
+        "* Deploy a Vertex AI Matching Engine Index and Brute Force Index\n",
+        "* Perform online queries\n",
+        "* Submit batch queries\n",
+        "* Compute recall metric"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0aaef374550b"
+      },
+      "source": [
+        "### Dataset\n",
         "\n",
+        "The dataset used for this tutorial is the [GloVe dataset](https://nlp.stanford.edu/projects/glove/)."
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5e2eba58ad71"
+      },
+      "source": [
         "### Costs \n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
@@ -87,6 +120,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "S5zc4kbEiYCm"
@@ -94,79 +128,47 @@
       "source": [
         "## Before you begin\n",
         "\n",
-        "* **Prepare a VPC network**.  To reduce any network overhead that might lead to unnecessary increase in overhead latency, it is best to call the ANN endpoints from your VPC via a direct [VPC Peering](https://cloud.google.com/vertex-ai/docs/general/vpc-peering) connection. The following section describes how to setup a VPC Peering connection if you don't have one. This is a one-time initial setup task. You can also reuse existing VPC network and skip this section.\n",
-        "* **WARNING:** The match service gRPC API (to create online queries against your deployed index) has to be executed in a Google Cloud Notebook instance that is created with the following requirements:\n",
-        "  * **In the same region as where your ANN service is deployed** (for example, if you set `REGION = \"us-central1\"` as same as the tutorial, the notebook instance has to be in `us-central1`).\n",
-        "  * **Make sure you select the VPC network you created for ANN service** (instead of using the \"default\" one). That is, you will have to create the VPC network below and then create a new notebook instance that uses that VPC.  \n",
-        "  * If you run it in the colab or a Google Cloud Notebook instance in a different VPC network or region, the gRPC API will fail to peer the network (InactiveRPCError)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "lW2LneA5mmmP"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_ID = \"<your_project_id>\"  # @param {type:\"string\"}\n",
-        "NETWORK_NAME = \"ucaip-haystack-vpc-network\"  # @param {type:\"string\"}\n",
-        "PEERING_RANGE_NAME = \"ucaip-haystack-range\"\n",
+        "### Set up your Google Cloud project\n",
         "\n",
-        "# Create a VPC network\n",
-        "! gcloud compute networks create {NETWORK_NAME} --bgp-routing-mode=regional --subnet-mode=auto --project={PROJECT_ID}\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "# Add necessary firewall rules\n",
-        "! gcloud compute firewall-rules create {NETWORK_NAME}-allow-icmp --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow icmp\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
         "\n",
-        "! gcloud compute firewall-rules create {NETWORK_NAME}-allow-internal --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow all --source-ranges 10.128.0.0/9\n",
+        "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "! gcloud compute firewall-rules create {NETWORK_NAME}-allow-rdp --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow tcp:3389\n",
+        "3. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
         "\n",
-        "! gcloud compute firewall-rules create {NETWORK_NAME}-allow-ssh --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow tcp:22\n",
-        "\n",
-        "# Reserve IP range\n",
-        "! gcloud compute addresses create {PEERING_RANGE_NAME} --global --prefix-length=16 --network={NETWORK_NAME} --purpose=VPC_PEERING --project={PROJECT_ID} --description=\"peering range for uCAIP Haystack.\"\n",
-        "\n",
-        "# Set up peering with service networking\n",
-        "! gcloud services vpc-peerings connect --service=servicenetworking.googleapis.com --network={NETWORK_NAME} --ranges={PEERING_RANGE_NAME} --project={PROJECT_ID}"
+        "4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk)."
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
-        "id": "d3uj8x73nDX_"
-      },
-      "source": [
-        "* Authentication: `$ gcloud auth login` rerun this in Google Cloud Notebook terminal when you are logged out and need the credential again."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "i7EUnXsZhAGF"
+        "id": "4700b0e39c5d"
       },
       "source": [
         "### Installation\n",
         "\n",
-        "Download and install the latest (preview) version of the Vertex SDK for Python."
+        "Download and install the latest version of the Vertex AI SDK for Python."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "wyy5Lbnzg5fi"
+        "id": "014470c6a8de"
       },
       "outputs": [],
       "source": [
-        "! pip install -U git+https://github.com/googleapis/python-aiplatform.git@main-test --user"
+        "! pip install -U git+https://github.com/googleapis/python-aiplatform.git@main --user"
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
-        "id": "irSMQn6gZ19l"
+        "id": "cf00462144f7"
       },
       "source": [
         "Install the `h5py` to prepare sample dataset, and the `grpcio-tools` for querying against the index. "
@@ -176,11 +178,15 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "-h5sqwOEZ5Yq"
+        "id": "3f3e45e5a1d1"
       },
       "outputs": [],
       "source": [
-        "! pip install -U grpcio-tools --user\n",
+        "! pip install protobuf==3.20.*\n",
+        "! pip install -U google-api-python-client==1.8.0 --user\n",
+        "! pip install -U grpcio-tools==1.47.0 --user\n",
+        "! pip install -U grpcio==1.47.0 --user\n",
+        "! pip install -U grpcio-status==1.47.0 --user\n",
         "! pip install -U h5py --user"
       ]
     },
@@ -199,7 +205,7 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "EzrelQZ22IZj"
+        "id": "aa1d87bdc90b"
       },
       "outputs": [],
       "source": [
@@ -215,79 +221,216 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
-        "id": "BF1j6f9HApxa"
+        "id": "249da91c1011"
       },
       "source": [
-        "### Set up your Google Cloud project\n",
+        "### Set your project ID\n",
         "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager).\n",
-        "\n",
-        "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "1. [Enable the Vertex AI API and Compute Engine API, and Service Networking API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,servicenetworking.googleapis.com).\n",
-        "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
-        "Cloud SDK uses the right project for all the commands in this notebook.\n",
-        "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "WReHDGG5g0XY"
-      },
-      "source": [
-        "#### Set your project ID\n",
-        "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "oM1iC_MfAts1"
+        "id": "10e0d2ee8c45"
       },
       "outputs": [],
       "source": [
-        "import os\n",
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
         "\n",
-        "PROJECT_ID = \"\"\n",
-        "\n",
-        "# Get your Google Cloud project ID from gcloud\n",
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
-        "    PROJECT_ID = shell_output[0]\n",
-        "    print(\"Project ID: \", PROJECT_ID)"
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
-        "id": "qJYoRfYng0XZ"
+        "id": "3fbfae3ff12a"
       },
       "source": [
-        "Otherwise, set your project ID here."
+        "### Set the region\n",
+        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations).\n",
+        "* **WARNING:** \n",
+        "    *  **Make sure to [choose a region where Vertex AI services are available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions).**\n",
+        "    *  **If you use Vertex Workbench, the Notebook instance needs to be in the same region where your Vertex AI Matching Engine is deployed.** (for example, if you set `REGION = \"us-central1\"` as same as the tutorial, the notebook instance has to be in `us-central1`)."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "riG_qUokg0XZ"
+        "id": "71c3fd82024e"
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
-        "    PROJECT_ID = \"<your_project_id>\"  # @param {type:\"string\"}"
+        "REGION = \"us-central1\"  # @param {type: \"string\"}\n",
+        "\n",
+        "# Set the regions\n",
+        "! gcloud config set ai_platform/region {REGION}"
       ]
     },
     {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "60c5a0f69ad8"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d118c95af93f"
+      },
+      "source": [
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3035286fcdda"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "455882ec0f11"
+      },
+      "outputs": [],
+      "source": [
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5097f3233d53"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2b88e46ac2c8"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fcdbb8929927"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7c6eef70dfdb"
+      },
+      "source": [
+        "### Prepare a VPC network\n",
+        "\n",
+        "To reduce any network overhead that might lead to unnecessary increase in overhead latency, it is best to call the Vertex AI Matching Engine endpoints from your VPC via a direct [VPC Peering](https://cloud.google.com/vertex-ai/docs/general/vpc-peering) connection. The following section describes how to setup a VPC Peering connection if you don't have one. This is a one-time initial setup task. You can also reuse existing VPC network and skip this section.\n",
+        "\n",
+        "* **WARNING:** The match service gRPC API (to create online queries against your deployed index) has to be executed in a Google Cloud Notebook instance that is created with the following requirements:\n",
+        "    * **Make sure you select the VPC network you created for Vertex AI Matching Engine service** (instead of using the \"default\" one). That is, you will have to create the VPC network below and then create a new notebook instance that uses that VPC.  \n",
+        "    * If you run it in the colab or a Google Cloud Notebook instance in a different VPC network or region, the gRPC API will fail to peer the network (InactiveRPCError)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ab38a8cc634c"
+      },
+      "outputs": [],
+      "source": [
+        "NETWORK_NAME = \"ucaip-haystack-vpc-network\"  # @param {type:\"string\"}\n",
+        "PEERING_RANGE_NAME = \"ucaip-haystack-range\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ec6bf3199835"
+      },
+      "outputs": [],
+      "source": [
+        "# Create a VPC network\n",
+        "! gcloud compute networks create {NETWORK_NAME} --bgp-routing-mode=regional --subnet-mode=auto --project={PROJECT_ID}\n",
+        "\n",
+        "# Add necessary firewall rules\n",
+        "! gcloud compute firewall-rules create {NETWORK_NAME}-allow-icmp --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow icmp\n",
+        "\n",
+        "! gcloud compute firewall-rules create {NETWORK_NAME}-allow-internal --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow all --source-ranges 10.128.0.0/9\n",
+        "\n",
+        "! gcloud compute firewall-rules create {NETWORK_NAME}-allow-rdp --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow tcp:3389\n",
+        "\n",
+        "! gcloud compute firewall-rules create {NETWORK_NAME}-allow-ssh --network {NETWORK_NAME} --priority 65534 --project {PROJECT_ID} --allow tcp:22\n",
+        "\n",
+        "# Reserve IP range\n",
+        "! gcloud compute addresses create {PEERING_RANGE_NAME} --global --prefix-length=16 --network={NETWORK_NAME} --purpose=VPC_PEERING --project={PROJECT_ID} --description=\"peering range for uCAIP Haystack.\""
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ddbace09fe81"
+      },
+      "source": [
+        "Create the VPC Peering. If you are running this from Vertex AI Workbench it is possible you might need your notebook's instance service or user account to have the Service Networking Admin Role"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d329aa3c54d3"
+      },
+      "outputs": [],
+      "source": [
+        "# Set up peering with service networking\n",
+        "! gcloud services vpc-peerings connect --service=servicenetworking.googleapis.com --network={NETWORK_NAME} --ranges={PEERING_RANGE_NAME} --project={PROJECT_ID}"
+      ]
+    },
+    {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "zgPO1eR3CYjk"
@@ -297,13 +440,11 @@
         "\n",
         "**The following steps are required, regardless of your notebook environment.**\n",
         "\n",
-        "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
+        "Create a storage bucket to store intermediate artifacts such as datasets. Set the name of your Cloud Storage bucket below. It must be unique across all\n",
         "Cloud Storage buckets.\n",
         "\n",
-        "You may also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n",
-        "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n",
-        "not use a Multi-Regional Storage bucket for training with Vertex AI."
+        "* **WARNING:** \n",
+        "    * **You may not use a Multi-Regional Storage bucket for training with Vertex AI.**"
       ]
     },
     {
@@ -314,8 +455,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}\n",
-        "REGION = \"us-central1\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"gs://[your-bucket-name-unique]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -328,10 +468,14 @@
       "source": [
         "from datetime import datetime\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
+        "UUID = datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
         "\n",
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if (\n",
+        "    BUCKET_NAME == \"\"\n",
+        "    or BUCKET_NAME is None\n",
+        "    or BUCKET_NAME == \"gs://[your-bucket-name-unique]\"\n",
+        "):\n",
+        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + UUID"
       ]
     },
     {
@@ -351,7 +495,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_NAME"
       ]
     },
     {
@@ -416,10 +560,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"\n",
         "ENDPOINT = \"{}-aiplatform.googleapis.com\".format(REGION)\n",
-        "NETWORK_NAME = \"ucaip-haystack-vpc-network\"  # @param {type:\"string\"}\n",
-        "\n",
         "\n",
         "AUTH_TOKEN = !gcloud auth print-access-token\n",
         "PROJECT_NUMBER = !gcloud projects list --filter=\"PROJECT_ID:'{PROJECT_ID}'\" --format='value(PROJECT_NUMBER)'\n",
@@ -429,10 +570,7 @@
         "\n",
         "print(\"ENDPOINT: {}\".format(ENDPOINT))\n",
         "print(\"PROJECT_ID: {}\".format(PROJECT_ID))\n",
-        "print(\"REGION: {}\".format(REGION))\n",
-        "\n",
-        "!gcloud config set project {PROJECT_ID}\n",
-        "!gcloud config set ai_platform/region {REGION}"
+        "print(\"REGION: {}\".format(REGION))"
       ]
     },
     {
@@ -523,12 +661,13 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "QuVl8DrWG8NS"
       },
       "source": [
-        "Upload the training data to GCS."
+        "Upload the training data to Google Cloud Storage"
       ]
     },
     {
@@ -539,9 +678,9 @@
       },
       "outputs": [],
       "source": [
-        "# NOTE: Everything in this GCS DIR will be DELETED before uploading the data.\n",
+        "# NOTE: Everything in this Google Cloud Storage directory will be DELETED before uploading the data\n",
         "\n",
-        "! gsutil rm -rf {BUCKET_NAME}/*"
+        "! gsutil rm -raf {BUCKET_NAME}/** 2> /dev/null || true"
       ]
     },
     {
@@ -567,21 +706,23 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "mglUPwHpJH98"
       },
       "source": [
-        "## Create Indexes\n"
+        "## Create the indexes\n"
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "qhIBCQ7dDSbW"
       },
       "source": [
-        "### Create ANN Index (for Production Usage)"
+        "### Create Vertex AI Matching Engine index (for production usage)"
       ]
     },
     {
@@ -597,6 +738,16 @@
         ")"
       ]
     },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "14e1ed031d66"
+      },
+      "source": [
+        "Set constants"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -611,14 +762,15 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "svLYiDf0OD2G"
       },
       "source": [
-        "Create the ANN index configuration:\n",
+        "#### Create the Vertex AI Matching Engine index configuration\n",
         "\n",
-        "Please read the documentation to understand the various configuration parameters that can be used to tune the index\n"
+        "Please read the [documentation](https://cloud.google.com/vertex-ai/docs/matching-engine/configuring-indexes) to understand the various configuration parameters that can be used to tune the index"
       ]
     },
     {
@@ -656,9 +808,9 @@
         "    }\n",
         ")\n",
         "\n",
-        "ann_index = {\n",
+        "matching_engine_index = {\n",
         "    \"display_name\": DISPLAY_NAME,\n",
-        "    \"description\": \"Glove 100 ANN index\",\n",
+        "    \"description\": \"Glove 100 Vertex AI Matching Engine Index\",\n",
         "    \"metadata\": struct_pb2.Value(struct_value=metadata),\n",
         "}"
       ]
@@ -671,7 +823,9 @@
       },
       "outputs": [],
       "source": [
-        "ann_index = index_client.create_index(parent=PARENT, index=ann_index)"
+        "matching_engine_index = index_client.create_index(\n",
+        "    parent=PARENT, index=matching_engine_index\n",
+        ")"
       ]
     },
     {
@@ -686,7 +840,7 @@
         "# This will take ~45 min.\n",
         "\n",
         "while True:\n",
-        "    if ann_index.done():\n",
+        "    if matching_engine_index.done():\n",
         "        break\n",
         "    print(\"Poll the operation to create index...\")\n",
         "    time.sleep(60)"
@@ -700,17 +854,18 @@
       },
       "outputs": [],
       "source": [
-        "INDEX_RESOURCE_NAME = ann_index.result().name\n",
+        "INDEX_RESOURCE_NAME = matching_engine_index.result().name\n",
         "INDEX_RESOURCE_NAME"
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "kSsqZuyoA1SG"
       },
       "source": [
-        "### Create Brute Force Index (for Ground Truth)\n",
+        "### Create brute force index (for ground truth)\n",
         "\n",
         "The brute force index uses a naive brute force method to find the nearest neighbors. This method is not fast or efficient. Hence brute force indices are not recommended for production usage. They are to be used to find the \"ground truth\" set of neighbors, so that the \"ground truth\" set can be used to measure recall of the indices being tuned for production usage. To ensure an apples to apples comparison, the `distanceMeasureType` and `featureNormType`, `dimensions` of the brute force index should match those of the production indices being tuned.\n",
         "\n",
@@ -725,8 +880,6 @@
       },
       "outputs": [],
       "source": [
-        "from google.protobuf import *\n",
-        "\n",
         "algorithmConfig = struct_pb2.Struct(\n",
         "    fields={\"bruteForceConfig\": struct_pb2.Value(struct_value=struct_pb2.Struct())}\n",
         ")\n",
@@ -796,12 +949,13 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "mglUPwHpJH98"
       },
       "source": [
-        "## Update Indexes\n",
+        "## Update the indexes\n",
         "\n",
         "Create incremental data file.\n"
       ]
@@ -863,10 +1017,10 @@
         "    }\n",
         ")\n",
         "\n",
-        "ann_index = {\n",
+        "matching_engine_index = {\n",
         "    \"name\": INDEX_RESOURCE_NAME,\n",
         "    \"display_name\": DISPLAY_NAME,\n",
-        "    \"description\": \"Glove 100 ANN index\",\n",
+        "    \"description\": \"Glove 100 Vertex AI Matching Engine Index\",\n",
         "    \"metadata\": struct_pb2.Value(struct_value=metadata),\n",
         "}"
       ]
@@ -879,7 +1033,7 @@
       },
       "outputs": [],
       "source": [
-        "ann_index = index_client.update_index(index=ann_index)"
+        "matching_engine_index = index_client.update_index(index=matching_engine_index)"
       ]
     },
     {
@@ -894,7 +1048,7 @@
         "# This will take ~45 min.\n",
         "\n",
         "while True:\n",
-        "    if ann_index.done():\n",
+        "    if matching_engine_index.done():\n",
         "        break\n",
         "    print(\"Poll the operation to update index...\")\n",
         "    time.sleep(60)"
@@ -908,17 +1062,18 @@
       },
       "outputs": [],
       "source": [
-        "INDEX_RESOURCE_NAME = ann_index.result().name\n",
+        "INDEX_RESOURCE_NAME = matching_engine_index.result().name\n",
         "INDEX_RESOURCE_NAME"
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "qV2xjAnDDObD"
       },
       "source": [
-        "## Create an IndexEndpoint with VPC Network"
+        "## Create an index endpoint with VPC network"
       ]
     },
     {
@@ -997,21 +1152,23 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "np2cgVuuIe9k"
       },
       "source": [
-        "## Deploy Indexes"
+        "## Deploy the indexes"
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "8Ew1UgcIIiJG"
       },
       "source": [
-        "### Deploy ANN Index"
+        "### Deploy a Vertex AI Matching Engine index"
       ]
     },
     {
@@ -1022,7 +1179,7 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOYED_INDEX_ID = \"ann_glove_deployed\""
+        "DEPLOYED_INDEX_ID = \"matching_engine_glove_deployed\""
       ]
     },
     {
@@ -1033,13 +1190,23 @@
       },
       "outputs": [],
       "source": [
-        "deploy_ann_index = {\n",
+        "deploy_matching_engine_index = {\n",
         "    \"id\": DEPLOYED_INDEX_ID,\n",
         "    \"display_name\": DEPLOYED_INDEX_ID,\n",
         "    \"index\": INDEX_RESOURCE_NAME,\n",
         "}"
       ]
     },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cb6d956d7419"
+      },
+      "source": [
+        "If errors occur with the next command wait some minutes for the index endpoint to be created and retry."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -1049,7 +1216,7 @@
       "outputs": [],
       "source": [
         "r = index_endpoint_client.deploy_index(\n",
-        "    index_endpoint=INDEX_ENDPOINT_NAME, deployed_index=deploy_ann_index\n",
+        "    index_endpoint=INDEX_ENDPOINT_NAME, deployed_index=deploy_matching_engine_index\n",
         ")"
       ]
     },
@@ -1082,12 +1249,13 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "RNZnXmO5AhDO"
       },
       "source": [
-        "### Deploy Brute Force Index"
+        "### Deploy brute force index"
       ]
     },
     {
@@ -1158,12 +1326,13 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "6LCGvBNvBd8D"
       },
       "source": [
-        "## Create Online Queries\n",
+        "## Create online queries\n",
         "\n",
         "After you built your indexes, you may query against the deployed index through the online querying gRPC API (Match service) within the virtual machine instances from the same region (for example 'us-central1' in this tutorial).  \n",
         "\n",
@@ -1178,7 +1347,15 @@
         "\n",
         "* Compile the protocal buffer (see below)\n",
         "* Obtain the index endpoint\n",
-        "* Use a code-generated stub to make the call, passing the parameter values"
+        "* Use a code-generated stub to make the call, passing the parameter values\n",
+        "\n",
+        "### Troubleshooting connectivity issues\n",
+        "\n",
+        "In case you have connectivity errors please perform the following:\n",
+        "\n",
+        "* Verify that the index endpoint, index, and VPC are all in the same Google Cloud project\n",
+        "* Verify that the index endpoint, index, and VPC are all in the same region and it is a valid (e.g. us-central1)\n",
+        "* Verify the Network does not have a firewall rule which denies all egress connections. Else, disable this rule or overwrite it with another rule that allows connection to the index endpoint IP"
       ]
     },
     {
@@ -1351,12 +1528,13 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "8wXTSgz1Bl0x"
       },
       "source": [
-        "Obtain the Private Endpoint: "
+        "Obtain the private endpoint: "
       ]
     },
     {
@@ -1521,12 +1699,13 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "_mNwdU9_B_Ez"
       },
       "source": [
-        "### Batch Query\n",
+        "## Submit a batch query\n",
         "\n",
         "You can run multiple queries in a single RPC call using the BatchMatch API:"
       ]
@@ -1764,18 +1943,20 @@
         "]\n",
         "\n",
         "batch_request = match_service_pb2.BatchMatchRequest()\n",
-        "batch_request_ann = match_service_pb2.BatchMatchRequest.BatchMatchRequestPerIndex()\n",
+        "batch_request_matching_engine = (\n",
+        "    match_service_pb2.BatchMatchRequest.BatchMatchRequestPerIndex()\n",
+        ")\n",
         "batch_request_brute_force = (\n",
         "    match_service_pb2.BatchMatchRequest.BatchMatchRequestPerIndex()\n",
         ")\n",
-        "batch_request_ann.deployed_index_id = DEPLOYED_INDEX_ID\n",
+        "batch_request_matching_engine.deployed_index_id = DEPLOYED_INDEX_ID\n",
         "batch_request_brute_force.deployed_index_id = DEPLOYED_BRUTE_FORCE_INDEX_ID\n",
         "for query in queries:\n",
-        "    batch_request_ann.requests.append(get_request(query, DEPLOYED_INDEX_ID))\n",
+        "    batch_request_matching_engine.requests.append(get_request(query, DEPLOYED_INDEX_ID))\n",
         "    batch_request_brute_force.requests.append(\n",
         "        get_request(query, DEPLOYED_BRUTE_FORCE_INDEX_ID)\n",
         "    )\n",
-        "batch_request.requests.append(batch_request_ann)\n",
+        "batch_request.requests.append(batch_request_matching_engine)\n",
         "batch_request.requests.append(batch_request_brute_force)\n",
         "\n",
         "response = stub.BatchMatch(batch_request)\n",
@@ -1783,14 +1964,15 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "_mNwdU9_B_Ez"
       },
       "source": [
-        "### Compute Recall\n",
+        "### Compute the recall metric\n",
         "\n",
-        "Use deployed brute force Index as the ground truth to calculate the recall of ANN Index:"
+        "Use the deployed brute force index as the ground truth to calculate the recall of the Vertex AI Matching Engine index:"
       ]
     },
     {
@@ -1835,6 +2017,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "TpV-iwP9qw9c"
@@ -1844,7 +2027,18 @@
         "\n",
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
-        "You can also manually delete resources that you created by running the following code."
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "390c331dc7d9"
+      },
+      "source": [
+        "### Delete the Vertex AI Matching Engine resources"
       ]
     },
     {
@@ -1869,6 +2063,31 @@
       "source": [
         "index_endpoint_client.delete_index_endpoint(name=INDEX_ENDPOINT_NAME)"
       ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ff14a85c85fb"
+      },
+      "source": [
+        "### Delete the Google Cloud Storage bucket"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "68d4781faac4"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "delete_bucket = False\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil -m rm -r $BUCKET_NAME"
+      ]
     }
   ],
   "metadata": {

notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb

@@ -1028,6 +1028,9 @@
         "deployment_resource_pool.dedicated_resources.min_replica_count = MIN_NODES\n",
         "deployment_resource_pool.dedicated_resources.max_replica_count = MAX_NODES\n",
         "deployment_resource_pool.dedicated_resources.machine_spec.machine_type = DEPLOY_COMPUTE\n",
+        "if DEPLOY_NGPU:\n",
+        "    deployment_resource_pool.dedicated_resources.machine_spec.accelerator_type = DEPLOY_GPU\n",
+        "    deployment_resource_pool.dedicated_resources.machine_spec.accelerator_count = DEPLOY_NGPU\n",
         "\n",
         "request = aip_beta.CreateDeploymentResourcePoolRequest(\n",
         "    parent=f\"projects/{PROJECT_ID}/locations/{REGION}\",\n",

notebooks/community/model_registry/vertex_ai_model_registry_bqml_custom_model_versioning.ipynb

@@ -198,7 +198,7 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "! pip3 install --upgrade tensorflow google-cloud-bigquery google-cloud-aiplatform {USER_FLAG} -q --no-warn-conflicts"
+        "! pip3 install --upgrade tensorflow google-cloud-bigquery google-cloud-aiplatform \"shapely<2\" {USER_FLAG} -q --no-warn-conflicts"
       ]
     },
     {

notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_anomaly_detection.ipynb

@@ -72,7 +72,7 @@
       "source": [
         "## Overview\n",
         "\n",
-        "Anomaly detection is the identification of rare obesrvations which deviate significantly from the data using ML. Anomaly detection can be done in many ways. Supervised, unsupervised, graph-based. It is particularly important for certain industries like telecommunications, manufacturing, and financial services.\n",
+        "Anomaly detection is the identification of rare observations which deviate significantly from the data using ML. Anomaly detection can be done in many ways. Supervised, unsupervised, graph-based. It is particularly important for certain industries like telecommunications, manufacturing, and financial services.\n",
         "\n",
         "For instance, in a manufacturing scenario, you may collect some sensor data to predict the number remaining cycles before engine failure (TTF). In this way, you can take actionable decisions about maintenance planning."
       ]
@@ -397,13 +397,12 @@
       "source": [
         "import os\n",
         "\n",
-        "SRC_PATH = \"src\"\n",
         "KFP_COMPONENTS_PATH = \"components\"\n",
         "PIPELINES_PATH = \"pipelines\"\n",
         "TRAIN_PIPELINES_PATH = os.path.join(PIPELINES_PATH, \"train_pipelines\")\n",
         "TEST_PIPELINES_PATH = os.path.join(PIPELINES_PATH, \"test_pipelines\")\n",
         "\n",
-        "! mkdir -m 777 -p {SRC_PATH} {KFP_COMPONENTS_PATH} {TRAIN_PIPELINES_PATH} {TEST_PIPELINES_PATH}"
+        "! mkdir -m 777 -p {KFP_COMPONENTS_PATH} {TRAIN_PIPELINES_PATH} {TEST_PIPELINES_PATH}"
       ]
     },
     {
@@ -425,14 +424,12 @@
       },
       "outputs": [],
       "source": [
-        "from urllib.parse import urlparse\n",
-        "\n",
         "PUBLIC_DATA_URI = (\n",
-        "    \" gs://cloud-samples-data/vertex-ai/pipeline-deployment/datasets/turbofan_anomaly\"\n",
+        "    \"gs://cloud-samples-data/vertex-ai/pipeline-deployment/datasets/turbofan_anomaly\"\n",
         ")\n",
-        "GCS_TRAIN_URI = urlparse(PUBLIC_DATA_URI)._replace(path=\"train_FD001.csv\").geturl()\n",
-        "GCS_TEST_URI = urlparse(PUBLIC_DATA_URI)._replace(path=\"test_FD001.csv\").geturl()\n",
-        "GCS_LABELS_URI = urlparse(PUBLIC_DATA_URI)._replace(path=\"RUL_FD001.csv\").geturl()"
+        "GCS_TRAIN_URI = f\"{PUBLIC_DATA_URI}/train_FD001.csv\"\n",
+        "GCS_TEST_URI = f\"{PUBLIC_DATA_URI}/test_FD001.csv\"\n",
+        "GCS_LABELS_URI = f\"{PUBLIC_DATA_URI}/RUL_FD001.csv\""
       ]
     },
     {
@@ -1478,7 +1475,6 @@
         "# Remove local resorces\n",
         "delete_local_resources = False\n",
         "if delete_local_resources:\n",
-        "    ! rm -rf {SRC_PATH}\n",
         "    ! rm -rf {KFP_COMPONENTS_PATH}\n",
         "    ! rm -rf {TRAIN_PIPELINES_PATH}\n",
         "    ! rm -rf {TEST_PIPELINES_PATH}"

notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb

@@ -54,18 +54,20 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "tvgnzT1CKxrO"
+        "id": "239ba71252d3"
       },
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook shows how to use `Vertex AI Pipelines` and `BigQuery ML pipeline components` to train and evaluate a demand forecasting model.\n",
-        "\n",
-        "### Dataset\n",
-        "\n",
-        "The dataset is a modified version of the dataset in [Build and visualize demand forecast predictions using Datastream, Dataflow, BigQuery ML, and Looker\n",
-        "](https://cloud.google.com/architecture/build-visualize-demand-forecast-prediction-datastream-dataflow-bigqueryml-looker) solution architecture\n",
-        "\n",
+        "This notebook shows how to use `Vertex AI Pipelines` and `BigQuery ML pipeline components` to train and evaluate a demand forecasting model."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "25c28706c23e"
+      },
+      "source": [
         "### Objective\n",
         "\n",
         "In this tutorial, you learn how to train and evaluate a BigQuery ML model using Vertex AI Pipelines and BigQuery ML pipeline components. \n",
@@ -87,8 +89,27 @@
         "  - Generate the ARIMA Plus forecasts\n",
         "  - Generate the ARIMA PLUS forecast explainations\n",
         "- Compile the pipeline.\n",
-        "- Execute the pipeline.\n",
+        "- Execute the pipeline."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "586acfa9b502"
+      },
+      "source": [
+        "### Dataset\n",
         "\n",
+        "The dataset is a modified version of the dataset in [Build and visualize demand forecast predictions using Datastream, Dataflow, BigQuery ML, and Looker\n",
+        "](https://cloud.google.com/architecture/build-visualize-demand-forecast-prediction-datastream-dataflow-bigqueryml-looker) solution architecture\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
         "### Costs \n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
@@ -352,9 +373,8 @@
         "id": "06571eb4063b"
       },
       "source": [
-        "#### Timestamp\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
+        "#### UUID\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -365,9 +385,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -485,7 +512,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"-aip-\" + TIMESTAMP\n",
+        "    BUCKET_NAME = PROJECT_ID + \"-aip-\" + UUID\n",
         "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
@@ -706,6 +733,7 @@
         "KFP_COMPONENTS_PATH = \"components\"\n",
         "PIPELINES_PATH = \"pipelines\"\n",
         "\n",
+        "! mkdir -m 777 -p {DATA_PATH}\n",
         "! mkdir -m 777 -p {KFP_COMPONENTS_PATH}\n",
         "! mkdir -m 777 -p {PIPELINES_PATH}"
       ]
@@ -771,7 +799,7 @@
         "  --location={LOCATION} \\\n",
         "  --source_format=CSV \\\n",
         "  --skip_leading_rows=1\\\n",
-        "  fast_fresh.orders_{TIMESTAMP} \\\n",
+        "  fast_fresh.orders_{UUID} \\\n",
         "  {RAW_DATA_URI} \\\n",
         "  time_of_sale:DATETIME,order_id:INTEGER,product_name:STRING,price:NUMERIC,quantity:NUMERIC,payment_method:STRING,store_id:INTEGER,user_id:INTEGER"
       ]
@@ -782,7 +810,7 @@
         "id": "ZrgOD30o7HcL"
       },
       "source": [
-        "## BQML Training Formalization\n",
+        "## BigQuery ML Training Formalization\n",
         "\n",
         "In the next cells, you build the components and pipeline to train and evaluate the BQML demand forecasting model."
       ]
@@ -820,13 +848,13 @@
         "BQ_EVALUATE_MODEL_TABLE_PREFIX = \"orders_arima_model_evaluate\"\n",
         "BQ_FORECAST_TABLE_PREFIX = \"orders_arima_forecast\"\n",
         "BQ_EXPLAIN_FORECAST_TABLE_PREFIX = \"orders_arima_explain_forecast\"\n",
-        "BQ_ORDERS_TABLE = f\"{BQ_ORDERS_TABLE_PREFIX}_{TIMESTAMP}\"\n",
-        "BQ_TRAINING_TABLE = f\"{BQ_TRAINING_TABLE_PREFIX}_{TIMESTAMP}\"\n",
-        "BQ_MODEL_TABLE = f\"{BQ_MODEL_TABLE_PREFIX}_{TIMESTAMP}\"\n",
-        "BQ_EVALUATE_TS_TABLE = f\"{BQ_EVALUATE_TS_TABLE_PREFIX}_{TIMESTAMP}\"\n",
-        "BQ_EVALUATE_MODEL_TABLE = f\"{BQ_EVALUATE_MODEL_TABLE_PREFIX}_{TIMESTAMP}\"\n",
-        "BQ_FORECAST_TABLE = f\"{BQ_FORECAST_TABLE_PREFIX}_{TIMESTAMP}\"\n",
-        "BQ_EXPLAIN_FORECAST_TABLE = f\"{BQ_EXPLAIN_FORECAST_TABLE_PREFIX}_{TIMESTAMP}\"\n",
+        "BQ_ORDERS_TABLE = f\"{BQ_ORDERS_TABLE_PREFIX}_{UUID}\"\n",
+        "BQ_TRAINING_TABLE = f\"{BQ_TRAINING_TABLE_PREFIX}_{UUID}\"\n",
+        "BQ_MODEL_TABLE = f\"{BQ_MODEL_TABLE_PREFIX}_{UUID}\"\n",
+        "BQ_EVALUATE_TS_TABLE = f\"{BQ_EVALUATE_TS_TABLE_PREFIX}_{UUID}\"\n",
+        "BQ_EVALUATE_MODEL_TABLE = f\"{BQ_EVALUATE_MODEL_TABLE_PREFIX}_{UUID}\"\n",
+        "BQ_FORECAST_TABLE = f\"{BQ_FORECAST_TABLE_PREFIX}_{UUID}\"\n",
+        "BQ_EXPLAIN_FORECAST_TABLE = f\"{BQ_EXPLAIN_FORECAST_TABLE_PREFIX}_{UUID}\"\n",
         "\n",
         "BQ_TRAIN_CONFIGURATION = {\n",
         "    \"destinationTable\": {\n",
@@ -1022,7 +1050,7 @@
         "id": "pcSL1FHk69KT"
       },
       "source": [
-        "### Build the BQML training pipeline\n",
+        "### Build the BigQuery ML training pipeline\n",
         "\n",
         "Define your workflow using Kubeflow Pipelines DSL package. \n",
         "\n",
@@ -1094,8 +1122,8 @@
         "        location=location,\n",
         "    ).set_display_name(\"get train data\")\n",
         "\n",
-        "    # Train the ARIMA PLUS model\n",
-        "    bq_arima_model_op = (\n",
+        "    # Run an ARIMA PLUS experiment\n",
+        "    bq_arima_model_exp_op = (\n",
         "        BigqueryCreateModelJobOp(\n",
         "            query=f\"\"\"\n",
         "        -- create model table\n",
@@ -1104,10 +1132,7 @@
         "        MODEL_TYPE = \\'ARIMA_PLUS\\',\n",
         "        TIME_SERIES_TIMESTAMP_COL = \\'hourly_timestamp\\',\n",
         "        TIME_SERIES_DATA_COL = \\'total_sold\\',\n",
-        "        TIME_SERIES_ID_COL = [\\'product_name\\'],\n",
-        "        MODEL_REGISTRY = \\'vertex_ai\\',\n",
-        "        VERTEX_AI_MODEL_ID = \\'order_demand_forecasting\\',\n",
-        "        VERTEX_AI_MODEL_VERSION_ALIASES = [\\'staging\\']\n",
+        "        TIME_SERIES_ID_COL = [\\'product_name\\']\n",
         "        ) AS\n",
         "        SELECT\n",
         "          hourly_timestamp,\n",
@@ -1119,7 +1144,7 @@
         "            project=project,\n",
         "            location=location,\n",
         "        )\n",
-        "        .set_display_name(\"train arima plus model\")\n",
+        "        .set_display_name(\"run arima+ model experiment\")\n",
         "        .after(create_training_dataset_op)\n",
         "    )\n",
         "\n",
@@ -1128,12 +1153,12 @@
         "        BigqueryMLArimaEvaluateJobOp(\n",
         "            project=project,\n",
         "            location=location,\n",
-        "            model=bq_arima_model_op.outputs[\"model\"],\n",
+        "            model=bq_arima_model_exp_op.outputs[\"model\"],\n",
         "            show_all_candidate_models=False,\n",
         "            job_configuration_query=bq_evaluate_time_series_configuration,\n",
         "        )\n",
         "        .set_display_name(\"evaluate arima plus time series\")\n",
-        "        .after(bq_arima_model_op)\n",
+        "        .after(bq_arima_model_exp_op)\n",
         "    )\n",
         "\n",
         "    # Evaluate ARIMA Plus model\n",
@@ -1141,12 +1166,12 @@
         "        BigqueryEvaluateModelJobOp(\n",
         "            project=project,\n",
         "            location=location,\n",
-        "            model=bq_arima_model_op.outputs[\"model\"],\n",
+        "            model=bq_arima_model_exp_op.outputs[\"model\"],\n",
         "            query_statement=f\"\"\"SELECT * FROM `{project}.{bq_dataset}.{bq_training_table}` WHERE split='TEST'\"\"\",\n",
         "            job_configuration_query=bq_evaluate_model_configuration,\n",
         "        )\n",
         "        .set_display_name(\"evaluate arima plus model\")\n",
-        "        .after(bq_arima_model_op)\n",
+        "        .after(bq_arima_model_exp_op)\n",
         "    )\n",
         "\n",
         "    # Plot model metrics\n",
@@ -1164,6 +1189,34 @@
         "        < PERF_THRESHOLD,\n",
         "        name=\"avg. mae good\",\n",
         "    ):\n",
+        "        # Train the ARIMA PLUS model\n",
+        "        bq_arima_model_op = (\n",
+        "            BigqueryCreateModelJobOp(\n",
+        "                query=f\"\"\"\n",
+        "        -- create model table\n",
+        "        CREATE OR REPLACE MODEL `{project}.{bq_dataset}.{bq_model_table}`\n",
+        "        OPTIONS(\n",
+        "        MODEL_TYPE = \\'ARIMA_PLUS\\',\n",
+        "        TIME_SERIES_TIMESTAMP_COL = \\'hourly_timestamp\\',\n",
+        "        TIME_SERIES_DATA_COL = \\'total_sold\\',\n",
+        "        TIME_SERIES_ID_COL = [\\'product_name\\'],\n",
+        "        MODEL_REGISTRY = \\'vertex_ai\\',\n",
+        "        VERTEX_AI_MODEL_ID = \\'order_demand_forecasting\\',\n",
+        "        VERTEX_AI_MODEL_VERSION_ALIASES = [\\'staging\\']\n",
+        "        ) AS\n",
+        "        SELECT\n",
+        "          DATETIME_TRUNC(time_of_sale, HOUR) as hourly_timestamp,\n",
+        "          product_name,\n",
+        "          SUM(quantity) AS total_sold,\n",
+        "          FROM `{project}.{bq_dataset}.{bq_orders_table}`\n",
+        "        GROUP BY hourly_timestamp, product_name;\n",
+        "        \"\"\",\n",
+        "                project=project,\n",
+        "                location=location,\n",
+        "            )\n",
+        "            .set_display_name(\"train arima+ model\")\n",
+        "            .after(get_evaluation_model_metrics_op)\n",
+        "        )\n",
         "\n",
         "        # Generate the ARIMA PLUS forecasts\n",
         "        bq_arima_forecast_op = (\n",
@@ -1224,7 +1277,7 @@
       "source": [
         "### Execute your pipeline\n",
         "\n",
-        "Next, you execute the pipeline. It takes the following parameters which we set as default:\n",
+        "Next, we execute the pipeline. It takes the following parameters which we set as default:\n",
         "\n",
         "- `bq_dataset`: The BigQuery dataset to train on.\n",
         "- `bq_orders_table` : The BigQuery table of raw data.\n",
@@ -1266,7 +1319,7 @@
       "source": [
         "### View BigQuery ML training pipeline results\n",
         "\n",
-        "Finally, you will view the artifact outputs of each task in the pipeline."
+        "Finally, you view the artifact outputs of each task in the pipeline."
       ]
     },
     {
@@ -1342,8 +1395,8 @@
         "print(\"bigquery-ml-arima-evaluate-job\")\n",
         "artifacts = print_pipeline_output(bqml_pipeline, \"bigquery-ml-arima-evaluate-job\")\n",
         "print(\"\\n\\n\")\n",
-        "print(\"get-model-evaluation-metrics\")\n",
-        "artifacts = print_pipeline_output(bqml_pipeline, \"get-model-evaluation-metrics\")\n",
+        "print(\"bigquery-evaluate-model-job\")\n",
+        "artifacts = print_pipeline_output(bqml_pipeline, \"bigquery-evaluate-model-job\")\n",
         "print(\"\\n\\n\")\n",
         "print(\"bigquery-forecast-model-job\")\n",
         "artifacts = print_pipeline_output(bqml_pipeline, \"bigquery-forecast-model-job\")\n",
@@ -1407,7 +1460,8 @@
         "\n",
         "# Remove local resorces\n",
         "! rm -rf {KFP_COMPONENTS_PATH}\n",
-        "! rm -rf {PIPELINES_PATH}"
+        "! rm -rf {PIPELINES_PATH}\n",
+        "! rm -rf {DATA_PATH}"
       ]
     }
   ],

notebooks/community/pipelines/google_cloud_pipeline_components_cloud_natural_language_pipeline.ipynb

@@ -0,0 +1,870 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "view-in-github"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/Narwhalprime/vertex-ai-samples/blob/main/notebooks/community/pipelines/google_cloud_pipeline_components_cloud_natural_language_pipeline.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1142fd18"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2022 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BwO30Ag12YcB"
+      },
+      "source": [
+        "# Vertex Pipelines: Cloud Natural Language model training pipeline\n",
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/pipelines/google_cloud_pipeline_components_cloud_natural_language_pipeline.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/pipelines/google_cloud_pipeline_components_cloud_natural_language_pipeline.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/natural_language/cloud_natural_language_pipeline.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "This notebook shows how to use [Google Cloud Pipeline Components SDK](https://cloud.google.com/vertex-ai/docs/pipelines/components-introduction) and additional components in this directory to run a machine learning pipeline in [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) to train a TensorFlow text classification model.\n",
+        "\n",
+        "In this pipeline, the model training Docker image utilizes [TFHub](https://tfhub.dev/) models to perform state-of-the-art text classification training. The image is pre-built and ready to use, so no additional Docker setup is required."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d975e698c9a4"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn how to construct an end-to-end training pipeine within Vertex AI pipelines that ingests a dataset, trains a text classification model on it, and outputs evaluation metrics.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Pipelines\n",
+        "- Vertex AI Datasets\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Define Kubeflow pipeline components\n",
+        "- Setup Kubeflow pipeline\n",
+        "- Run pipeline on Vertex AI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "08d289fa873f"
+      },
+      "source": [
+        "## Dataset\n",
+        "\n",
+        "This notebook requires that the user has two datasets exported from Vertex AI [managed datasets](https://cloud.google.com/vertex-ai/docs/training/using-managed-datasets): one with train and validation data splits, and the other with test data used for evaluation. Please ensure no data is shared between the two datasets (in particular, no evaluation data should be part of the train or validation splits). To export a Vertex AI dataset, please follow the following public docs:\n",
+        "* [Preparing data](https://cloud.google.com/vertex-ai/docs/text-data/classification/prepare-data)\n",
+        "* [Creating a Vertex AI dataset](https://cloud.google.com/vertex-ai/docs/text-data/classification/create-dataset) from the above data\n",
+        "* [Exporting dataset and its annotations](https://cloud.google.com/vertex-ai/docs/datasets/export-metadata-annotations); ensure the resulting export is located in a Google Cloud Storage (GCS) bucket you own. You may need to manually separate the test split data into its own file."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aed92deeb4a0"
+      },
+      "source": [
+        "## Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "setup_local"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "If you are using Colab or Google Vertex AI Workbench Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "\n",
+        "***NOTE***: This notebook has been tested in the following environment:\n",
+        "\n",
+        "* Python version = 3.8\n",
+        "\n",
+        "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
+        "\n",
+        "- The Cloud Storage SDK\n",
+        "- Python 3\n",
+        "- virtualenv\n",
+        "- Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:\n",
+        "\n",
+        "1. [Install and initialize the SDK](https://cloud.google.com/sdk/docs/).\n",
+        "\n",
+        "2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).\n",
+        "\n",
+        "3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "\n",
+        "4. Activate that environment and run `pip3 install Jupyter` in a terminal shell to install Jupyter.\n",
+        "\n",
+        "5. Run `jupyter notebook` on the command line in a terminal shell to launch Jupyter.\n",
+        "\n",
+        "6. Open this notebook in the Jupyter Notebook Dashboard.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "568d5c16"
+      },
+      "source": [
+        "### Install additional packages\n",
+        "\n",
+        "Run the following commands to setup the packages for this notebook. Note that the last code snippet in this section restarts your kernel in order to load the installs properly, so when initalizing this notebook from scratch, it is recommended to run up to that cell, then afterwards you may start running the cell after that."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dac98aac"
+      },
+      "outputs": [],
+      "source": [
+        "# Install using pip3\n",
+        "!pip3 install -U tensorflow google-cloud-pipeline-components google-cloud-aiplatform kfp==1.8.16 \"shapely<2\" -q"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "alRWYgYTdz7P"
+      },
+      "outputs": [],
+      "source": [
+        "# Version check\n",
+        "# This has been tested with KFP 1.8.16\n",
+        "! python3 -c \"import kfp; print('KFP SDK version: {}'.format(kfp.__version__))\"\n",
+        "! python3 -c \"import google_cloud_pipeline_components; print('google_cloud_pipeline_components version: {}'.format(google_cloud_pipeline_components.__version__))\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d0a15440"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "B9IYalYObAbY"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "3. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,storage.googleapis.com).\n",
+        "\n",
+        "4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "VA_kzAIIj2G_"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. Skip this step.\n",
+        "\n",
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PyQmSRbKA8r-"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "set_service_account"
+      },
+      "source": [
+        "### Set project ID\n",
+        "\n",
+        "Set your project ID here. If you don't know this, the following snippet attempts to deterine this from your gcloud config. Please continue only if the notebook can see your desired project."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "AkqEd5Gin9mn"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"your-project-id\"  # @param {type:\"string\"}\n",
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "print(\"Project ID:\", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OVO_gUqpFEP2"
+      },
+      "outputs": [],
+      "source": [
+        "!gcloud config set project $PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a27d4cee"
+      },
+      "source": [
+        "### Setup project information\n",
+        "\n",
+        "Enter information about your project and datasets here."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7e9477a2"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us\"  # @param {type:\"string\"}\n",
+        "LOCATION = \"us-central1\"  # @param {type:\"string\"}\n",
+        "TRAINING_DATA_LOCATION = \"gs://your-training-data-location\"  # @param {type:\"string\"}\n",
+        "TASK_TYPE = \"CLASSIFICATION\"  # @param [\"CLASSIFICATION\", \"MULTILABEL_CLASSIFICATION\"]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "o-MZnHsimbOH"
+      },
+      "outputs": [],
+      "source": [
+        "# Since we are training a custom model, we need to specify the list of possible\n",
+        "# classes/labels.\n",
+        "# e.g, [\"FirstClass\", \"SecondClass\"]\n",
+        "# An additional class \"[UNK]\" will be added to the list indicating that none of\n",
+        "# the specified labels are a match.\n",
+        "CLASS_NAMES = [\"\"]\n",
+        "\n",
+        "# This is a list of GCS URIs; e.g., [\"gs://your-bucket-name-here/your-input-file.jsonl\"].\n",
+        "TEST_DATA_URIS = [\"gs://your-bucket-name-here/your-input-file.jsonl\"]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "timestamp"
+      },
+      "source": [
+        "#### UUID\n",
+        "\n",
+        "To avoid name collisions with other resources in your project, you can create a UUID with the code below and append it onto the name of the bucket(s) created in this notebook."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "wh9sgzemwLXE"
+      },
+      "outputs": [],
+      "source": [
+        "import random\n",
+        "import string\n",
+        "\n",
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bucket:mbsdk"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "When you initialize the Vertex AI SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
+        "\n",
+        "Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bucket"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "autoset_bucket"
+      },
+      "outputs": [],
+      "source": [
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "create_bucket"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dO0NV93IwLXF"
+      },
+      "outputs": [],
+      "source": [
+        "!gsutil mb -l $REGION $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "validate_bucket"
+      },
+      "source": [
+        "Finally, validate access to your Cloud Storage bucket by examining its contents:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Hg5f2oKBwLXG"
+      },
+      "outputs": [],
+      "source": [
+        "!gsutil ls -al $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EuFETRptyKXc"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform\n",
+        "\n",
+        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f3a09765"
+      },
+      "source": [
+        "## Create training pipeline"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "89bb4a50"
+      },
+      "source": [
+        "### Import libraries"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0f361e65"
+      },
+      "outputs": [],
+      "source": [
+        "from google_cloud_pipeline_components.aiplatform import ModelBatchPredictOp\n",
+        "from google_cloud_pipeline_components.experimental import natural_language\n",
+        "from google_cloud_pipeline_components.experimental.evaluation import (\n",
+        "    GetVertexModelOp, ModelEvaluationClassificationOp,\n",
+        "    TargetFieldDataRemoverOp)\n",
+        "from kfp import components\n",
+        "from kfp.v2 import compiler, dsl"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d33c87e4-2ada-4b87-bf75-064247f3162d"
+      },
+      "source": [
+        "### Define constants"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "36ceb9f8"
+      },
+      "outputs": [],
+      "source": [
+        "# Worker pool specs\n",
+        "TRAINING_MACHINE_TYPE = \"n1-highmem-8\"\n",
+        "ACCELERATOR_TYPE = \"NVIDIA_TESLA_T4\"\n",
+        "ACCELERATOR_COUNT = 1\n",
+        "EVAL_MACHINE_TYPE = \"n1-highmem-8\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zAaMJKrhAe5L"
+      },
+      "source": [
+        "## Define components\n",
+        "\n",
+        "This pipeline is composed from the following components:\n",
+        "\n",
+        "- **train-tfhub-model** - Trains a new Tensorflow model using TFHub layers from pre-built Docker image\n",
+        "- **upload-tensorflow-model-to-google-cloud-vertex-ai** - Uploads resulting model to Vertex AI model registry\n",
+        "- **get-vertex-model** - Gets model that has just been uploaded as an artifact in pipeline\n",
+        "- **convert-dataset-export-for-batch-predict** - Preprocessing component that takes the test dataset exported from Vertex datasets and converts it to a simpler compatible one that is readable from the batch predict component\n",
+        "- **target-field-data-remover** - Removes the target field (i.e., label) in the test dataset for the downstream batch predict component\n",
+        "- **model-batch-predict** - Performs a batch prediction job\n",
+        "- **model-evaluation-classification** - Calculates the evaluation metrics from the above batch predict job and exports the metrics artifact\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "DKe2iQNKgpKG"
+      },
+      "outputs": [],
+      "source": [
+        "# Load upload TF model component\n",
+        "upload_tensorflow_model_to_vertex_op = components.load_component_from_url(\n",
+        "    \"https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml\"\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TEnh9Pcx6Xfi"
+      },
+      "source": [
+        "### Define the pipeline\n",
+        "\n",
+        "The pipeline performs the following steps:\n",
+        "- Trains new text classification model\n",
+        "- Uploads model to Vertex AI Model Registry\n",
+        "- Performs preprocessing steps on test dataset export: formats data for batch predcition, removes target field\n",
+        "- Performs batch prediction on preprocessed test data\n",
+        "- Evaluates performance of model based on batch prediction output"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2a67cde8"
+      },
+      "outputs": [],
+      "source": [
+        "@dsl.pipeline(name=\"text-classification-model\")\n",
+        "def pipeline():\n",
+        "    train_task = natural_language.TrainTextClassificationOp()(\n",
+        "        project=PROJECT_ID,\n",
+        "        location=LOCATION,\n",
+        "        machine_type=TRAINING_MACHINE_TYPE,\n",
+        "        accelerator_type=ACCELERATOR_TYPE,\n",
+        "        accelerator_count=ACCELERATOR_COUNT,\n",
+        "        input_data_path=TRAINING_DATA_LOCATION,\n",
+        "        input_format=\"jsonl\",\n",
+        "        natural_language_task_type=TASK_TYPE,\n",
+        "    )\n",
+        "\n",
+        "    upload_task = upload_tensorflow_model_to_vertex_op(\n",
+        "        model=train_task.outputs[\"model_output\"]\n",
+        "    )\n",
+        "\n",
+        "    get_model_task = GetVertexModelOp(\n",
+        "        model_resource_name=upload_task.outputs[\"model_name\"]\n",
+        "    )\n",
+        "\n",
+        "    classification_type = (\n",
+        "        \"multilabel\" if TASK_TYPE == \"MULTILABEL_CLASSIFICATION\" else \"multiclass\"\n",
+        "    )\n",
+        "\n",
+        "    convert_dataset_task = natural_language.ConvertDatasetExportForBatchPredictOp(\n",
+        "        file_paths=TEST_DATA_URIS, classification_type=classification_type\n",
+        "    )\n",
+        "\n",
+        "    target_field_remover_task = TargetFieldDataRemoverOp(\n",
+        "        project=PROJECT_ID,\n",
+        "        location=LOCATION,\n",
+        "        root_dir=BUCKET_URI,\n",
+        "        gcs_source_uris=convert_dataset_task.outputs[\"output_files\"],\n",
+        "        target_field_name=\"labels\",\n",
+        "        instances_format=\"jsonl\",\n",
+        "    )\n",
+        "\n",
+        "    # Note: ModelBatchPredictOp doesn't support accelerators currently.\n",
+        "    batch_predict_task = ModelBatchPredictOp(\n",
+        "        project=PROJECT_ID,\n",
+        "        location=LOCATION,\n",
+        "        model=get_model_task.outputs[\"model\"],\n",
+        "        job_display_name=\"nl-batch-predict-evaluation\",\n",
+        "        gcs_source_uris=target_field_remover_task.outputs[\"gcs_output_directory\"],\n",
+        "        instances_format=\"jsonl\",\n",
+        "        predictions_format=\"jsonl\",\n",
+        "        gcs_destination_output_uri_prefix=BUCKET_URI,\n",
+        "        machine_type=EVAL_MACHINE_TYPE,\n",
+        "    )\n",
+        "\n",
+        "    # Note: Because we're running a custom training pipeline, the model source\n",
+        "    # is detected as Custom and thus it doesn't use AutoML NL's default settings\n",
+        "    # and fails if class_labels is excluded.\n",
+        "    ModelEvaluationClassificationOp(\n",
+        "        project=PROJECT_ID,\n",
+        "        location=LOCATION,\n",
+        "        root_dir=BUCKET_URI,\n",
+        "        class_labels=CLASS_NAMES + [\"[UNK]\"],\n",
+        "        predictions_gcs_source=batch_predict_task.outputs[\"gcs_output_directory\"],\n",
+        "        predictions_format=\"jsonl\",\n",
+        "        prediction_label_column=\"prediction.displayNames\",\n",
+        "        prediction_score_column=\"prediction.confidences\",\n",
+        "        ground_truth_gcs_source=convert_dataset_task.outputs[\"output_files\"],\n",
+        "        ground_truth_format=\"jsonl\",\n",
+        "        target_field_name=\"labels\",\n",
+        "        classification_type=TASK_TYPE,\n",
+        "    )"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3211ba19"
+      },
+      "source": [
+        "### Compile the pipeline"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c368c73f"
+      },
+      "outputs": [],
+      "source": [
+        "compiler.Compiler().compile(pipeline, \"nl_pipeline.json\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "l_Vxwz5cdF5f"
+      },
+      "source": [
+        "Running the above line will generate a file locally or in Colab's directory."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ax0jOxIaholy"
+      },
+      "source": [
+        "### Run the pipeline\n",
+        "\n",
+        "This sends a create pipeline job request to Vertex Pipelines. Note that this  task run synchronously and may take a while to complete.\n",
+        "\n",
+        "You may view the progress of the job at any time by clicking on the generated links (after \"View Pipeline Job\" in the console output of the cell below). Once the pipeline finishes, you may examine the artifacts produced from this pipeline."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Wfs7QOSxhp_n"
+      },
+      "outputs": [],
+      "source": [
+        "job = aiplatform.PipelineJob(\n",
+        "    display_name=\"nl_pipeline\",\n",
+        "    template_path=\"nl_pipeline.json\",\n",
+        "    location=LOCATION,\n",
+        "    enable_caching=True,\n",
+        "    parameter_values={},\n",
+        ")\n",
+        "\n",
+        "job.run()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UIyGPaihWJWn"
+      },
+      "source": [
+        "Once the pipeline successfully finishes, go to the pipeline and examine the resulting metrics artifacts for the results. Otherwise, refer to the failing step(s) in the pipeline to determine the cause of any errors."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OoexTJTy9jnH"
+      },
+      "source": [
+        "## View model evaluation results\n",
+        "\n",
+        "To check the results of evaluation after pipeline execution, find the \"model-evaluation-classification\" subdirectory in the Cloud Storage bucket created by this pipeline. You may also run the following to directly output the contents of the metrics file:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "h9EqPCQF9lN9"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "\n",
+        "EVAL_TASK_NAME = \"model-evaluation-classification\"\n",
+        "PROJECT_NUMBER = job.gca_resource.name.split(\"/\")[1]\n",
+        "for _ in range(len(job.gca_resource.job_detail.task_details)):\n",
+        "    TASK_ID = job.gca_resource.job_detail.task_details[_].task_id\n",
+        "    EVAL_METRICS = (\n",
+        "        BUCKET_URI\n",
+        "        + \"/\"\n",
+        "        + PROJECT_NUMBER\n",
+        "        + \"/\"\n",
+        "        + job.name\n",
+        "        + \"/\"\n",
+        "        + EVAL_TASK_NAME\n",
+        "        + \"_\"\n",
+        "        + str(TASK_ID)\n",
+        "        + \"/executor_output.json\"\n",
+        "    )\n",
+        "    if tf.io.gfile.exists(EVAL_METRICS):\n",
+        "        ! gsutil cat $EVAL_METRICS"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TpV-iwP9qw9c"
+      },
+      "source": [
+        "## Cleaning up\n",
+        "\n",
+        "To clean up the resources used by this pipeline, run the command below:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "sx_vKniMq9ZX"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete GCS bucket.\n",
+        "!gsutil -m rm -r {BUCKET_URI}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UMuyzrnZLoUa"
+      },
+      "source": [
+        "# Next steps\n",
+        "\n",
+        "For an alternate approach, please check out the [\"ready-to-go\" text classification pipeline](https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/pipelines/google_cloud_pipeline_components_ready_to_go_text_classification_pipeline.ipynb). This pipeline exposes the model logic for further customization if needed, and adds an additional pipeline step to deploy the model to enable online predictions."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [
+        "d975e698c9a4",
+        "08d289fa873f",
+        "d33c87e4-2ada-4b87-bf75-064247f3162d",
+        "3211ba19",
+        "TpV-iwP9qw9c",
+        "UMuyzrnZLoUa"
+      ],
+      "name": "google_cloud_pipeline_components_cloud_natural_language_pipeline.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/community/vertex_endpoints/optimized_tensorflow_runtime/benchmark.py

@@ -112,7 +112,7 @@ def benchmark(
 
     results = []
     for qps in qps_list:
-        num_requests = max(qps * duration_sec, 10)
+        num_requests = int(max(qps * duration_sec, 10))
         requests_for_qps = list(
             itertools.islice(itertools.cycle(requests), num_requests)
         )

notebooks/community/vertex_endpoints/torchserve/dreambooth_stablediffusion.ipynb

@@ -0,0 +1,986 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "18ebbd838e32"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2023 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "219f1b1fe8fe"
+      },
+      "source": [
+        "# Deploy and host a Stable Diffusion model on Vertex AI\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JAPoU8Sm5E6e"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/vertex_endpoints/torchserve/dreambooth_stable_diffusion.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/vertex_endpoints/torchserve/dreambooth_stable_diffusion.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        " <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/vertex_endpoints/torchserve/dreambooth_stable_diffusion.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fce05a8186d6"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates how to deploy and host a fine-tuned [Stable Diffusion 1.5](https://huggingface.co/runwayml/stable-diffusion-v1-5) model on Vertex AI. For hosting, you use the PyTorch 3 container built for Vertex AI with [TorchServe](https://pytorch.org/serve/index.html)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c76216b03fec"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn how to host and deploy a Stable Diffusion 1.5 model on Vertex AI.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "+ Vertex AI `Model` resource\n",
+        "+ Vertex AI `Endpoint` resource\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "+ Create a `torchserve` handler for responding to prediction requests.\n",
+        "+ Upload a Stable Diffusion 1.5 model on a prebuilt PyTorch container in Vertex AI.\n",
+        "+ Deploy a model to a Vertex AI Endpoint.\n",
+        "+ Send requests to the endpoint and parse the responses using Vertex AI Prediction service."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c6deba5a8557"
+      },
+      "source": [
+        "### Model\n",
+        "\n",
+        "This notebook uses a collection of model artifacts fine-tuned to generate images of a small dog. These are the same images used in the original [DreamBooth paper](https://dreambooth.github.io/)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "911dc651ea9c"
+      },
+      "source": [
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI models\n",
+        "* Vertex AI endpoints\n",
+        "* Vertex AI prediction\n",
+        "* Cloud Storage\n",
+        "* (Optionally) Vertex AI Workbench\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0d36bd3d53fa"
+      },
+      "source": [
+        "## Hardware requirements\n",
+        "\n",
+        "This notebook requires that you use a GPU with a sufficient amount of VRAM available. It was tested on a `NVIDIA Tesla A100 GPU` with 85 GB of VRAM. Run the following cell to ensure that you have the correct hardware configuration."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3dd4022552e5"
+      },
+      "outputs": [],
+      "source": [
+        "!nvidia-smi --query-gpu=name,memory.total,memory.free --format=\"csv,noheader\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a782627e5f73"
+      },
+      "source": [
+        "### Create a user-managed notebook on Vertex AI\n",
+        "\n",
+        "If you are using Vertex AI Workbench, you can create a notebook with the correct configuration by doing the following:\n",
+        "\n",
+        "+ Go to [Vertex AI Workbench](https://console.cloud.google.com/vertex-ai/workbench/user-managed) in the Google Cloud Console.\n",
+        "+ Click **New Notebook** and then click **PyTorch 1.13** > **With 1 NVIDIA T4**.\n",
+        "+ In the **New notebook** dialog box, click **Advanced Options**. The **Create a user-managed notebook** page opens up.\n",
+        "+ In the **Create a user-managed notebook** page, do the following:\n",
+        "  * In the **Notebook name** box, type a name for your notebook, for example \"my-stablediffusion-nb\".\n",
+        "  * In the **Machine type** drop-down, select **A2 highgpu** > **a2-highgpu-1g**.\n",
+        "  * In the **GPU type** drop-down, select **NVIDIA Tesla A100**.\n",
+        "  * Check the box next to **Install NVIDIA GPU driver automatically for me**\n",
+        "  * Expand **Disk(s)** and do the following:\n",
+        "    - Under **Boot disk type**, select **SSD Persistent Disk**.\n",
+        "    - Under **Data disk type**, select **SSD Persistent Disk**.\n",
+        "  * Click **Create**."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FAPoU8Sm5E6e"
+      },
+      "source": [
+        "<div style=\"background:#feefe3; padding:5px; color:#aa0000\">\n",
+        "<strong>Caution:</strong> Using a Vertex AI Workbench notebook with the above configuration can increase your costs significantly. You can estimate your costs using the <a href=\"https://cloud.google.com/products/calculator\"><u>costs calculator</u></a>.</div>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0bb4201cc99a"
+      },
+      "source": [
+        "## Installation\n",
+        "\n",
+        "Install the following packages required to execute this notebook.\n",
+        "\n",
+        "**Note**: You might need to change the version of PyTorch (`torch`) installed by `pip`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9c769df171a6"
+      },
+      "outputs": [],
+      "source": [
+        "%%writefile requirements.txt\n",
+        "diffusers\n",
+        "ftfy\n",
+        "google-cloud-aiplatform\n",
+        "gradio\n",
+        "ninja\n",
+        "tensorboard==1.15.0\n",
+        "torch\n",
+        "torchaudio\n",
+        "torchvision\n",
+        "torchserve\n",
+        "torch-model-archiver\n",
+        "torch-workflow-archiver\n",
+        "transformers"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e46804ac90d8"
+      },
+      "outputs": [],
+      "source": [
+        "%pip install -r requirements.txt"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "58707a750154"
+      },
+      "source": [
+        "### Colab only: Uncomment the following cell to restart the kernel."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "77c11549298a"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
+        "\n",
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "294df346a918"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "3. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
+        "\n",
+        "4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oM1iC_MfAts1"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "region"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sBCra4QMA2wR"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "74ccc9e52986"
+      },
+      "source": [
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "de775a3773ba"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "254614fa0c46"
+      },
+      "outputs": [],
+      "source": [
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ef21552ccea8"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "603adbbf0532"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f6b2ccc891ed"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cb5c4ca3e851"
+      },
+      "source": [
+        "### Import libraries"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7348591eda51"
+      },
+      "outputs": [],
+      "source": [
+        "import base64\n",
+        "import math\n",
+        "\n",
+        "import torch\n",
+        "from diffusers import StableDiffusionPipeline\n",
+        "from google.cloud import aiplatform\n",
+        "from IPython import display\n",
+        "from PIL import Image\n",
+        "from torch import autocast"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "697566b5f660"
+      },
+      "source": [
+        "## Optional: View model inferences\n",
+        "\n",
+        "Before uploading the model to Vertex AI, you can review the expected output from the model. The model used in this notebook is available for your use and can be downloaded from Cloud Storage. This download may take a few minutes to complete."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d63df8d91215"
+      },
+      "outputs": [],
+      "source": [
+        "!gsutil -m cp gs://cloud-samples-data/vertex-ai/model-deployment/models/stable-diffusion/model_artifacts.zip \\\n",
+        "   ."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "89d613ae9573"
+      },
+      "outputs": [],
+      "source": [
+        "!unzip model_artifacts.zip"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9192ea4f3b57"
+      },
+      "source": [
+        "### Create new images\n",
+        "\n",
+        "With everything in place, you can now generate new images from the Stable Diffusion model. First you must load your model into a `StableDiffusionPipeline`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cd1f30223b79"
+      },
+      "outputs": [],
+      "source": [
+        "model_path = \"model_artifacts\"\n",
+        "\n",
+        "pipe = StableDiffusionPipeline.from_pretrained(\n",
+        "    model_path, torch_dtype=torch.float16\n",
+        ").to(\"cuda\")\n",
+        "\n",
+        "g_cuda = None"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dd91520f58cf"
+      },
+      "outputs": [],
+      "source": [
+        "g_cuda = torch.Generator(device=\"cuda\")\n",
+        "seed = 52362\n",
+        "g_cuda.manual_seed(seed)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1e9c5d734b9f"
+      },
+      "source": [
+        "With the model loaded into a `StableDiffusionPipeline`, you can now generate results (inferences) from the model. Each set of inference requires an input (called a [prompt](https://learnprompting.org/)) that specifies what the model should create.\n",
+        "\n",
+        "You can also vary other inputs into the model, as shown in the following cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a28b73de55ce"
+      },
+      "outputs": [],
+      "source": [
+        "prompt = \"photo of examplePup dog in a Monet style\"\n",
+        "\n",
+        "num_samples = 4\n",
+        "num_batches = 1\n",
+        "num_columns = 2\n",
+        "guidance_scale = 10\n",
+        "num_inference_steps = 50\n",
+        "height = 512\n",
+        "width = 512"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "11a7bb79de59"
+      },
+      "outputs": [],
+      "source": [
+        "def image_grid(imgs, cols):\n",
+        "    total = len(imgs)\n",
+        "    rows = math.ceil(total / cols)\n",
+        "\n",
+        "    w, h = imgs[0].size\n",
+        "    grid = Image.new(\"RGB\", size=(cols * w, rows * h))\n",
+        "    grid_w, grid_h = grid.size\n",
+        "\n",
+        "    for i, img in enumerate(imgs):\n",
+        "        grid.paste(img, box=(i % cols * w, i // cols * h))\n",
+        "    return grid\n",
+        "\n",
+        "\n",
+        "all_images = []\n",
+        "for _ in range(num_batches):\n",
+        "    with autocast(\"cuda\"):\n",
+        "        images = pipe(\n",
+        "            [prompt] * num_samples,\n",
+        "            height=height,\n",
+        "            width=width,\n",
+        "            num_inference_steps=num_inference_steps,\n",
+        "            guidance_scale=guidance_scale,\n",
+        "        ).images\n",
+        "        all_images.extend(images)\n",
+        "\n",
+        "\n",
+        "grid = image_grid(all_images, num_columns)\n",
+        "grid"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bbc72963ff88"
+      },
+      "source": [
+        "## Deploy the model to Vertex AI\n",
+        "\n",
+        "You can host your Stable Diffusion 1.5 model on a Vertex AI endpoint where you can get inferences from it online. Uploading your model is a four step process: \n",
+        "\n",
+        "1. Create a custom TorchServe handler.\n",
+        "1. Upload the model artifacts onto Cloud Storage.\n",
+        "2. Create a Vertex AI model with the model artifacts and a prebuilt PyTorch container image.\n",
+        "3. Deploy the Vertex AI model onto an endpoint."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eafbb0e0-40e6-43a0-a38e-edc54323da51"
+      },
+      "source": [
+        "### Create the custom TorchServe handler\n",
+        "\n",
+        "The model deployed to Vertex AI uses [TorchServe](https://pytorch.org/serve/) to handle requests and return responses from the model. You must create a custom TorchServe handler to include in with the model artifacts uploaded to Vertex AI.\n",
+        "\n",
+        "The handler file should be included in the directory with the other model artifacts."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "94567a87-9d74-4c87-a749-306ddaf01b61"
+      },
+      "outputs": [],
+      "source": [
+        "%%writefile model_artifacts/handler.py\n",
+        "\n",
+        "\"\"\"Customized handler for Stable Diffusion 1.5.\"\"\"\n",
+        "import base64\n",
+        "import logging\n",
+        "from io import BytesIO\n",
+        "\n",
+        "import torch\n",
+        "from diffusers import EulerDiscreteScheduler\n",
+        "from diffusers import StableDiffusionPipeline\n",
+        "from ts.torch_handler.base_handler import BaseHandler\n",
+        "\n",
+        "logger = logging.getLogger(__name__)\n",
+        "model_id = 'runwayml/stable-diffusion-v1-5'\n",
+        "\n",
+        "\n",
+        "class ModelHandler(BaseHandler):\n",
+        "\n",
+        "  def __init__(self):\n",
+        "    self.initialized = False\n",
+        "    self.map_location = None\n",
+        "    self.device = None\n",
+        "    self.use_gpu = True\n",
+        "    self.store_avg = True\n",
+        "    self.pipe = None\n",
+        "\n",
+        "  def initialize(self, context):\n",
+        "    \"\"\"Initializes the pipe.\"\"\"\n",
+        "    properties = context.system_properties\n",
+        "    gpu_id = properties.get('gpu_id')\n",
+        "\n",
+        "    self.map_location, self.device, self.use_gpu = \\\n",
+        "      ('cuda', torch.device('cuda:' + str(gpu_id)),\n",
+        "       True) if torch.cuda.is_available() else \\\n",
+        "        ('cpu', torch.device('cpu'), False)\n",
+        "\n",
+        "    # Use the Euler scheduler here instead\n",
+        "    scheduler = EulerDiscreteScheduler.from_pretrained(model_id,\n",
+        "                                                       subfolder='scheduler')\n",
+        "    pipe = StableDiffusionPipeline.from_pretrained(model_id,\n",
+        "                                                   scheduler=scheduler,\n",
+        "                                                   torch_dtype=torch.float16)\n",
+        "    pipe = pipe.to('cuda')\n",
+        "    # Uncomment the following line to reduce the GPU memory usage.\n",
+        "    # pipe.enable_attention_slicing()\n",
+        "    self.pipe = pipe\n",
+        "\n",
+        "    self.initialized = True\n",
+        "\n",
+        "  def preprocess(self, requests):\n",
+        "    \"\"\"Noting to do here.\"\"\"\n",
+        "    logger.info('requests: %s', requests)\n",
+        "    return requests\n",
+        "\n",
+        "  def inference(self, preprocessed_data, *args, **kwargs):\n",
+        "    \"\"\"Run the inference.\"\"\"\n",
+        "    images = []\n",
+        "    for pd in preprocessed_data:\n",
+        "      prompt = pd['prompt']\n",
+        "      images.extend(self.pipe(prompt).images)\n",
+        "    return images\n",
+        "\n",
+        "  def postprocess(self, output_batch):\n",
+        "    \"\"\"Converts the images to base64 string.\"\"\"\n",
+        "    postprocessed_data = []\n",
+        "    for op in output_batch:\n",
+        "      fp = BytesIO()\n",
+        "      op.save(fp, format='JPEG')\n",
+        "      postprocessed_data.append(base64.b64encode(fp.getvalue()).decode('utf-8'))\n",
+        "      fp.close()\n",
+        "    return postprocessed_data\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6ace1dac0af0"
+      },
+      "source": [
+        "After creating the handler file, you must package the handler as a model archiver (MAR) file. The output file must be named 'model.mar'."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "67707f95d440"
+      },
+      "outputs": [],
+      "source": [
+        "!torch-model-archiver \\\n",
+        "  -f \\\n",
+        "  --model-name model \\\n",
+        "  --version 1.0 \\\n",
+        "  --handler model_artifacts/handler.py \\\n",
+        "  --export-path model_artifacts"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ffab030f4bc8"
+      },
+      "source": [
+        "### Upload the model artifacts to Cloud Storage\n",
+        "\n",
+        "Create a new folder in your Cloud Storage bucket to hold the model artifacts"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MzGDU7TWdts_"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_NAME = \"your-bucket-name-unique\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}/\"\n",
+        "FULL_GCS_PATH = f\"{BUCKET_URI}model_artifacts\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-EcIXiGsCePi"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NIq7R4HZCfIc"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "971232e28657"
+      },
+      "source": [
+        "Next, upload the model archive file and your trained Stable Diffusion 1.5 model to the folder on Cloud Storage."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ef6baf44c808"
+      },
+      "outputs": [],
+      "source": [
+        "!gsutil cp -r model_artifacts $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "402370ca9396"
+      },
+      "source": [
+        "### Create the Vertex AI model\n",
+        "\n",
+        "Once you've uploaded the model artifacts into a Cloud Storage bucket, you can create a new Vertex AI model. This notebook uses the [Vertex AI SDK](https://cloud.google.com/vertex-ai/docs/start/use-vertex-ai-python-sdk) to create the model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b6c58a74a0fd"
+      },
+      "outputs": [],
+      "source": [
+        "PYTORCH_PREDICTION_IMAGE_URI = (\n",
+        "    \"us-docker.pkg.dev/vertex-ai/prediction/pytorch-gpu.1-12:latest\"\n",
+        ")\n",
+        "APP_NAME = \"my-stable-diffusion\"\n",
+        "VERSION = 1\n",
+        "MODEL_DISPLAY_NAME = \"stable_diffusion_1_5-unique\"\n",
+        "MODEL_DESCRIPTION = \"stable_diffusion_1_5 container\"\n",
+        "ENDPOINT_DISPLAY_NAME = f\"{APP_NAME}-endpoint\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "07c3503a1a2e"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_NAME)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FAPoU8Sm5E6e"
+      },
+      "source": [
+        "<div style=\"background:#e3effe; padding:5px; color:#0000aa\">\n",
+        "<strong>Note:</strong> The next cell fails if you haven't <a href=\"https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com\"><u>enabled the Vertex API</u></a>.</div>"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a776324dd16f"
+      },
+      "outputs": [],
+      "source": [
+        "model = aiplatform.Model.upload(\n",
+        "    display_name=MODEL_DISPLAY_NAME,\n",
+        "    description=MODEL_DESCRIPTION,\n",
+        "    serving_container_image_uri=PYTORCH_PREDICTION_IMAGE_URI,\n",
+        "    artifact_uri=FULL_GCS_PATH,\n",
+        ")\n",
+        "\n",
+        "model.wait()\n",
+        "\n",
+        "print(model.display_name)\n",
+        "print(model.resource_name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0fbc3d371574"
+      },
+      "source": [
+        "### Deploy the model to an endpoint\n",
+        "\n",
+        "To get online preductions from your Stable Diffusion 2.0 model, you must [deploy it to a Vertex AI endpoint](https://cloud.google.com/vertex-ai/docs/predictions/overview). You can again use the Vertex AI SDK to create the endpoint and deploy your model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ab29f0a770cb"
+      },
+      "outputs": [],
+      "source": [
+        "endpoint = aiplatform.Endpoint.create(display_name=ENDPOINT_DISPLAY_NAME)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "25f703df88c7"
+      },
+      "outputs": [],
+      "source": [
+        "model.deploy(\n",
+        "    endpoint=endpoint,\n",
+        "    deployed_model_display_name=MODEL_DISPLAY_NAME,\n",
+        "    machine_type=\"n1-standard-8\",\n",
+        "    accelerator_type=\"NVIDIA_TESLA_P100\",\n",
+        "    accelerator_count=1,\n",
+        "    traffic_percentage=100,\n",
+        "    deploy_request_timeout=1200,\n",
+        "    sync=True,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c9fc560df0a9"
+      },
+      "source": [
+        "The previous cell, which deploys your model to the endpoint, can take a while to complete. If the previous cell times out before returning, your endpoint might still be successfully deployed to an endpoint. Check the [Cloud Console](https://console.cloud.google.com/vertex-ai/endpoints) to verify the results.\n",
+        "\n",
+        "You can also extend the time to wait for deployment by changing the `deploy_request_timeout` argument passed to `model.deploy()`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "88a5304dfdc9"
+      },
+      "source": [
+        "## Get online predictions\n",
+        "\n",
+        "Finally, with your Stable Diffusion 1.5 model deployed to a Vertex AI endpoint, you can now get online predictions from it. Using the Vertex AI SDK, you only need a few lines of code to get an inference."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0d6bc4aa34d6"
+      },
+      "outputs": [],
+      "source": [
+        "instances = [{\"prompt\": \"An examplePup dog with a baseball jersey.\"}]\n",
+        "response = endpoint.predict(instances=instances)\n",
+        "\n",
+        "with open(\"img5.jpg\", \"wb\") as g:\n",
+        "    g.write(base64.b64decode(response.predictions[0]))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "65bafefda60c"
+      },
+      "outputs": [],
+      "source": [
+        "display.Image(\"img5.jpg\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TpV-iwP9qw9c"
+      },
+      "source": [
+        "## Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "sx_vKniMq9ZX"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# Delete endpoint resource\n",
+        "endpoint.undeploy_all()\n",
+        "endpoint.delete()\n",
+        "\n",
+        "# Delete model resource\n",
+        "model.delete()\n",
+        "\n",
+        "# Delete Cloud Storage objects that were created\n",
+        "delete_bucket = False\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil -m rm -r $BUCKET_URI"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "dreambooth_stablediffusion.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/notebook_template_review.py

@@ -12,6 +12,10 @@
         --errors-codes: A list of error codes to report errors. Otherwise, all errors are reported.
         --errors-csv: Report errors in CSV format
         
+        # options for automatic fixing
+        --fix: Automatic fix
+        --fix-codes: A list of fix codes to fix. Otherwise, all fix codes are enabled.
+        
         # index generatation
         --repo: Generate index in markdown format
         --web: Generate index in HTML format
@@ -19,6 +23,7 @@
         --desc: Add description to index
         --steps: Add steps to index
         --uses: Add "resources" used to index
+        --linkback: Add linkback to index
         
     Format of CSV file for notebooks to review:
     
@@ -61,6 +66,8 @@ parser.add_argument('--uses', dest='uses', action='store_true',
                     default=False, help='Output uses (resources)')
 parser.add_argument('--steps', dest='steps', action='store_true', 
                     default=False, help='Ouput steps')
+parser.add_argument('--linkback', dest='linkback', action='store_true', 
+                    default=False, help='Ouput linkback')
 parser.add_argument('--web', dest='web', action='store_true', 
                     default=False, help='Output format in HTML')
 parser.add_argument('--repo', dest='repo', action='store_true', 
@@ -109,13 +116,14 @@ class ErrorCode(Enum):
     #   Costs cell required
     #     Check for required Vertex and optional BQ and Dataflow
     ERROR_OVERVIEW_NOTFOUND = 10,
-    ERROR_OBJECTIVE_NOTFOUND = 11,
-    ERROR_OBJECTIVE_MISSING_DESC = 12,
-    ERROR_OBJECTIVE_MISSING_USES = 13,
-    ERROR_OBJECTIVE_MISSING_STEPS = 14,
-    ERROR_DATASET_NOTFOUND = 15,
-    ERROR_COSTS_NOTFOUND = 16,
-    ERROR_COSTS_MISSING = 17,
+    ERROR_LINKBACK_NOTFOUND = 11,
+    ERROR_OBJECTIVE_NOTFOUND = 12,
+    ERROR_OBJECTIVE_MISSING_DESC = 13,
+    ERROR_OBJECTIVE_MISSING_USES = 14,
+    ERROR_OBJECTIVE_MISSING_STEPS = 15,
+    ERROR_DATASET_NOTFOUND = 16,
+    ERROR_COSTS_NOTFOUND = 17,
+    ERROR_COSTS_MISSING = 18,
 
     # Installation cell
     #   Installation cell required
@@ -126,34 +134,34 @@ class ErrorCode(Enum):
     #   option {USER_FLAG} required
     #   installation code cell not match template
     #   all packages must be installed as a single pip3
-    ERROR_INSTALLATION_NOTFOUND = 18,
-    ERROR_INSTALLATION_HEADING = 19,
-    ERROR_INSTALLATION_CODE_NOTFOUND = 20,
-    ERROR_INSTALLATION_PIP3 = 21,
-    ERROR_INSTALLATION_QUIET = 22,
-    ERROR_INSTALLATION_USER_FLAG = 23,
-    ERROR_INSTALLATION_CODE_TEMPLATE = 24,
-    ERROR_INSTALLATION_SINGLE_PIP3 = 25,
+    ERROR_INSTALLATION_NOTFOUND = 19,
+    ERROR_INSTALLATION_HEADING = 20,
+    ERROR_INSTALLATION_CODE_NOTFOUND = 21,
+    ERROR_INSTALLATION_PIP3 = 22,
+    ERROR_INSTALLATION_QUIET = 23,
+    ERROR_INSTALLATION_USER_FLAG = 24,
+    ERROR_INSTALLATION_CODE_TEMPLATE = 25,
+    ERROR_INSTALLATION_SINGLE_PIP3 = 26,
 
     # Restart kernel cell
     #    Restart code cell required
     #    Restart code cell not found
-    ERROR_RESTART_NOTFOUND = 23,
-    ERROR_RESTART_CODE_NOTFOUND = 24,
+    ERROR_RESTART_NOTFOUND = 27,
+    ERROR_RESTART_CODE_NOTFOUND = 28,
 
     # Before you begin cell
     #    Before you begin cell required
     #    Before you begin cell incomplete
-    ERROR_BEFOREBEGIN_NOTFOUND = 25,
-    ERROR_BEFOREBEGIN_INCOMPLETE = 26,
+    ERROR_BEFOREBEGIN_NOTFOUND = 29,
+    ERROR_BEFOREBEGIN_INCOMPLETE = 30,
 
     # Set Project ID
     #    Set project ID cell required
     #    Set project ID code cell not found
     #    Set project ID not match template
-    ERROR_PROJECTID_NOTFOUND = 27,
-    ERROR_PROJECTID_CODE_NOTFOUND = 28,
-    ERROR_PROJECTID_TEMPLATE = 29,
+    ERROR_PROJECTID_NOTFOUND = 31,
+    ERROR_PROJECTID_CODE_NOTFOUND = 32,
+    ERROR_PROJECTID_TEMPLATE = 33,
 
     # Technical Writer Rules
     ERROR_TWRULE_TODO = 51,
@@ -182,7 +190,21 @@ def parse_dir(directory: str) -> int:
     """
     exit_code = 0
     
+    sorted_entries = []
     entries = os.scandir(directory)
+    for entry in entries:
+
+        inserted = False
+        for ix in range(len(sorted_entries)):
+            if entry.name < sorted_entries[ix].name:
+                sorted_entries.insert(ix, entry)
+                inserted = True
+                break
+        
+        if not inserted:
+            sorted_entries.append(entry)
+    
+    entries = sorted_entries
     for entry in entries:
         if entry.is_dir():
             if entry.name[0] == '.':
@@ -590,6 +612,8 @@ class OverviewRule(NotebookRule):
                 linkback = more.split('(')[1].split(')')[0]
                 self.tags.append(tag)
                 self.linkbacks.append(linkback)
+        else:
+            return notebook.report_error(ErrorCode.ERROR_LINKBACK_NOTFOUND, "Linkback missing in overview section")
                 
         return True
 
@@ -618,6 +642,10 @@ class ObjectiveRule(NotebookRule):
         in_steps = False
     
         for line in cell['source'][1:]:
+            # TOC anchor
+            if line.startswith('<a name='):
+                continue
+                
             if line.startswith('This tutorial uses'):
                 in_desc = False
                 in_steps = False
@@ -654,16 +682,39 @@ class ObjectiveRule(NotebookRule):
                         # check for italic font setting
                         if ch == '*' and sline[1] != ' ':
                             in_steps = False
+                        # special case
+                        elif sline.startswith('* Prediction Service'):
+                            in_steps = False
                         else:
                             self.steps += line
                     elif ch == '#':
                         in_steps = False
+
             
         if self.desc == '':
             ret = notebook.report_error(ErrorCode.ERROR_OBJECTIVE_MISSING_DESC, "Objective section missing desc")
         else:
             self.desc = self.desc.lstrip()
-            sentences = self.desc.split('.')
+            
+            bracket = False
+            paren = False
+            sentences = ""
+            for _ in range(len(self.desc)):
+                if self.desc[_] == '[':
+                    bracket = True
+                    continue
+                elif self.desc[_] == ']':
+                    bracket = False
+                    continue
+                elif self.desc[_] == '(':
+                    paren = True
+                elif self.desc[_] == ')':
+                    paren = False
+                    continue
+                    
+                if not paren:
+                    sentences += self.desc[_]
+            sentences = sentences.split('.')
             if len(sentences) > 1:
                 self.desc = sentences[0] + '.\n'
             if self.desc.startswith('In this tutorial, you learn') or self.desc.startswith('In this notebook, you learn'):
@@ -683,7 +734,7 @@ class ObjectiveRule(NotebookRule):
             ret = notebook.report_error(ErrorCode.ERROR_OBJECTIVE_MISSING_STEPS, "Objective section missing steps list")
             
         notebook.costs = self.costs
-        ret = True
+        return ret
 
 
 class RecommendationsRule(NotebookRule):
@@ -1080,48 +1131,81 @@ def add_index(path: str,
     title = title.split(':')[-1].strip()
     title = title[0].upper() + title[1:]
     if args.web:
-        title = title.replace('`', '')
+        title = replace_cl(title.replace('`', ''))
         
         print('    <tr>')
         print('        <td>')
         for tag in tags:
+            tag = replace_cl(tag)
             print(f'            {tag.strip()}<br/>\n')
         print('        </td>')
         print('        <td>')
-        print(f'            <b>{title}</b><br/>\n')
+        print(f'            <b>{title}</b>. ')
         if args.desc:
-            desc = desc.replace('`', '')
+            desc = replace_cl(desc.replace('`', ''))
             print('<br/>')
             print(f'            {desc}<br/>\n')
             
         if args.steps:
-            print('<br/>' + steps.replace('\n', '<br/>').replace('-', '&nbsp;&nbsp;-').replace('*', '&nbsp;&nbsp;-') +  '<br/>')
+            print("<devsite-expandable>\n")
+            print('  <h6 class="showalways">Notebook steps</h6>\n')
+            print('  <ul style="font-size: .75em">\n')
             
-        if linkbacks:
+            if ":" in steps:
+                steps = steps.split(':')[1].replace('*', '').replace('-', '').strip().split('\n')
+            else:
+                steps = []
+              
+            for step in steps:
+                print(f'    <li>{replace_cl(step)}</li>\n')
+            #steps = replace_cl(steps.replace('\n', '<br/>').replace('-', '&nbsp;&nbsp;-').replace('**', '').replace('*', '&nbsp;&nbsp;-').replace('`', ''))
+            #print('<br/>' + steps +  '<br/>')
+            print('  </ul>\n')
+            print("</devsite-expandable>\n")
+            
+        if args.linkback and linkbacks:
             num = len(tags)
             for _ in range(num):
                 if linkbacks[_].startswith("vertex-ai"):
-                    print(f'<br/>            Learn more about <a href="https://cloud.google.com/{linkbacks[_]}">{tags[_]}</a>\n')
+                    print(f' Learn more about <a href="https://cloud.google.com/{linkbacks[_]}." target="_blank">{replace_cl(tags[_])}</a>.\n')
                 else:
-                    print(f'<br/>            Learn more about <a href="{linkbacks[_]}">{tags[_]}</a>\n')
+                    print(f' Learn more about <a href="{linkbacks[_]}." target="_blank">{replace_cl(tags[_])}</a>.\n')
+                    
+        if args.steps:
+            print("<devsite-expandable>\n")
+            print('  <h6 class="showalways">Notebook steps</h6>\n')
+            print('  <ul>\n')
+            
+            if ":" in steps:
+                steps = steps.split(':')[1].replace('*', '').replace('-', '').strip().split('\n')
+            else:
+                steps = []
+              
+            for step in steps:
+                print(f'    <li>{replace_cl(step)}</li>\n')
+            print('  </ul>\n')
+            print("</devsite-expandable>\n")
                     
         print('        </td>')
         print('        <td>')
         if colab_link:
-            print(f'            <a href="{colab_link}" target="_blank">Colab</a><br/>\n')
+            print(f'            <a href="{colab_link}" target="_blank" class="external" track-type="notebookTutorial" track-name="colabLink">Colab</a><br/>\n')
         if git_link:
-            print(f'            <a href="{git_link}" target="_blank">GitHub</a><br/>\n')
+            print(f'            <a href="{git_link}" target="_blank" class="external" track-type="notebookTutorial" track-name="gitHubLink">GitHub</a><br/>\n')
         if workbench_link:
-            print(f'            <a href="{workbench_link}" target="_blank">Vertex AI Workbench</a><br/>\n')
+            print(f'            <a href="{workbench_link}" target="_blank" class="external" track-type="notebookTutorial" track-name="workbenchLink">Vertex AI Workbench</a><br/>\n')
         print('        </td>')
         print('    </tr>\n')
     elif args.repo:
-        if tags != last_tag and tag != '':
-            last_tag = tags
-            flat_list = ''
-            for item in tags:
-                flat_list += item.replace("'", '') + ' '
-            print(f"\n### {flat_list}\n")
+        try:
+            if tags != last_tag and tag != '':
+                last_tag = tags
+                flat_list = ''
+                for item in tags:
+                    flat_list += item.replace("'", '') + ' '
+                print(f"\n### {flat_list}\n")
+        except:
+            pass
         print(f"\n[{title}]({git_link})\n")
     
         print("```")
@@ -1133,7 +1217,80 @@ def add_index(path: str,
 
         if args.steps:
             print(steps.rstrip() + '\n')
+            
         print("```\n")
+            
+        if args.linkback and linkbacks:
+            num = len(tags)
+            for _ in range(num):
+                if linkbacks[_].startswith("vertex-ai"):
+                    print(f'&nbsp;&nbsp;&nbsp;Learn more about [{tags[_]}]({linkbacks[_]}).\n')
+                else:
+                    print(f'&nbsp;&nbsp;&nbsp;Learn more about [{tags[_]}]({linkbacks[_]}).\n')
+
+def replace_cl(text : str ) -> str:
+    '''
+    Replace product names with CL substitution variables
+    '''
+    substitutions = {
+        #'AutoML Tabular Workflow': '{{automl_name}} Tabular Workflow',
+        #'AutoML Tables': '{{automl_tables_name}}',
+        #'AutoML Tabular': '{{automl_tables_name}}',
+        #'AutoML Vision': '{automl_vision_name}}',
+        #'AutoML Image': '{automl_vision_name}}',
+        'AutoML': '{{automl_name}}',
+        
+        'BigQuery ML': '{{bigqueryml_name}}',
+        'BQML': '{{bigqueryml_name}}',
+        'BigQuery': '{{bigquery_name}}',
+        'BQ': '{{bigquery_name}}',
+        
+        'Vertex Dataset': '{{vertex_ai_name}} Dataset',
+        'Vertex Model': '{{vertex_ai_name}} Model',
+        'Vertex Endpoint': '{{vertex_ai_name}} Endpoint',
+        'Vertex Model Registry': '{{vertex_model_registry_name}}',
+        'Vertex AI Model Registry': '{{vertex_model_registry_name}}',
+        'Vertex Training': '{{vertex_training_name}}',
+        'Vertex AI Training': '{{vertex_training_name}}',
+        'Vertex Prediction': '{{vertex_prediction_name}}',
+        'Vertex AI Prediction': '{{vertex_prediction_name}}',
+        'Vertex TensorBoard': '{{vertex_tensorboard_name}}',
+        'Vertex AI TensorBoard': '{{vertex_tensorboard_name}}',
+        'Vertex ML Metadata': '{{vertex_metadata_name}}',
+        'Vertex Pipelines': '{{vertex_pipelines_name}}',
+        'Vertex AI Pipelines': '{{vertex_pipelines_name}}',
+        'Vertex AI Data Labeling': '{{vertex_data_labeling_name}}',
+        'Vertex AI Experiments': '{{vertex_experiments_name}}',
+        'Vertex Experiments': '{{vertex_experiments_name}}',
+        'Vertex AI Matching Engine': '{{vertex_matching_engine_name}}',
+        'Vertex Matching Engine': '{{vertex_matching_engine_name}}',
+        'Vertex Model Monitoring': '{{vertex_model_monitoring_name}}',
+        'Vertex AI Model Monitoring': '{{vertex_model_monitoring_name}}',
+        'Vertex Feature Store': '{{vertex_featurestore_name}}',
+        'Vertex AI Feature Store': '{{vertex_featurestore_name}}',
+        'Vertex Vizier': '{{vertex_vizier_name}}',
+        'Vertex AI Vizier': '{{vertex_vizier_name}}',
+        'Vertex Explainable AI': '{{vertex_xai_name}}',
+        'NAS': '{{vertex_nas_name}',
+        'Vertex AI Neural Architectural Search': '{{vertex_nas_name}}',
+        'Vertex Workbench': '{{vertex_workbench_name}}',
+        'Vertex AI Workbench': '{{vertex_workbench_name}}',
+        'Vertex AI Edge Manager': '{{vertex_edge_manager_name}}',
+        'Vertex SDK': '{{vertex_sdk_name}}',
+        'Vertex AI SDK': '{{vertex_sdk_name}}',
+        'Vertex AI': '{{vertex_ai_name}}',
+        
+        'Cloud Storage': '{{storage_name}}',
+        'TensorFlow Enterprise': '{{tf4gcp_name}}',
+        'TensorFlow': '{{tensorflow_name}}',
+    }
+    
+    for key, value in substitutions.items():
+        if key in text:
+            text = text.replace(key, value)
+            
+    return text
+
 
 
 # Instantiate the rules
@@ -1172,9 +1329,14 @@ if args.web:
     print('}')
     print('</style>')
     print('<table>')
-    print('    <th width="180px">Services</th>')
-    print('    <th>Description</th>')
-    print('    <th width="80px">Open in</th>')
+    print('    <thead>')
+    print('        <tr>')
+    print('            <th width="180px">Services</th>')
+    print('            <th>Description</th>')
+    print('            <th width="80px">Open in</th>')
+    print('        </tr>')
+    print('    </thead>')
+    print('    <tbody>')
 
 if args.notebook_dir:
     if not os.path.isdir(args.notebook_dir):
@@ -1210,6 +1372,7 @@ else:
     exit(1)
 
 if args.web:
+    print('    </tbody>\n')
     print('</table>\n')
 
 exit(exit_code)

notebooks/official/CODEOWNERS

@@ -29,15 +29,14 @@
 /pipelines/google_cloud_pipelines_dataproc_tabular @inardini
 /automl/automl_forecasting_bqml_arima_plus_comparison.ipynb @TheMichaelHu
 /automl/automl_tabular_on_vertex_pipelines.ipynb @helinwang
-/custom/custom_training_tensorboard_profiler.ipynb @itseric
+/custom/custom_training_tensorboard_profiler.ipynb @gericdong
+/custom/get_started_with_vertex_endpoint_and_shared_vm.ipynb @andrewferlitsch
 /workbench/spark/spark_sample_notebook.ipynb @bradmiro
 /workbench/spark/spark_ml.ipynb @bradmiro
 /model_registry/bqml_vertexai_model_registry.ipynb @soheilazangeneh
-/workbench/exploratory_data_analysis/explore_data_in_bigquery_with_workbench.ipynb @alokpattani
 /model_evaluation/automl_tabular_classification_model_evaluation.ipynb @soheilazangeneh
 /model_evaluation/automl_tabular_regression_model_evaluation.ipynb @soheilazangeneh
-/tabular_workflows/tabnet_on_vertex_pipelines.ipynb @sakagarwal
-/tabular_workflows/wide_and_deep_on_vertex_pipelines.ipynb @sakagarwal
+/tabular_workflows/prophet_on_vertex_pipelines.ipynb @TheMichaelHu
 /model_evaluation/custom_tabular_classification_model_evaluation.ipynb @soheilazangeneh
 /sdk/SDK_FBProphet_Forecasting_Online.ipynb @brianchunkang
 /automl/sdk_automl_forecasting_hierarchical_batch.ipynb @ivanmkc

notebooks/official/automl/README.md

@@ -1,4 +1,5 @@
-[AutoML Tabular Training and Prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl-tabular-classification.ipynb)
+
+[AutoML Tabular training and prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl-tabular-classification.ipynb)
 
 ```
 Learn how to train and make predictions on an AutoML model based on a tabular dataset.
@@ -13,6 +14,8 @@ The steps performed include the following:
 
 ```
 
+   Learn more about [Classification for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview).
+
 
 [Create, train, and deploy an AutoML text classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl-text-classification.ipynb)
 
@@ -31,97 +34,7 @@ The steps performed include:
 
 ```
 
-
-[AutoML training video classification model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_classification_batch.ipynb)
-
-```
-Learn how to create an AutoML video classification model from a Python script, and then do a batch prediction using the Vertex AI SDK.
-
-The steps performed include:
-
-- Create a Vertex `Dataset` resource.
-- Train the model.
-- View the model evaluation.
-- Make a batch prediction.
-
-```
-
-
-[AutoML training text entity extraction model for online prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_text_entity_extraction_online.ipynb)
-
-```
-Learn how to create an AutoML text entity extraction model and deploy for online prediction from a Python script using the Vertex SDK.
-
-The steps performed include:
-
-- Create a Vertex `Dataset` resource.
-- Train the model.
-- View the model evaluation.
-- Deploy the `Model` resource to a serving `Endpoint` resource.
-- Make a prediction.
-- Undeploy the `Model`.
-
-```
-
-
-[AutoML tabular forecasting model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_forecasting_batch.ipynb)
-
-```
-Learn how to create an `AutoML` tabular forecasting model from a Python script, and then do a batch prediction using the Vertex AI SDK.
-
-The steps performed include:
-
-- Create a `Vertex AI Dataset` resource.
-- Train an `AutoML` tabular forecasting `Model` resource.
-- Obtain the evaluation metrics for the `Model` resource.
-- Make a batch prediction.
-
-```
-
-
-[AutoML training video action recognition model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_action_recognition_batch.ipynb)
-
-```
-Learn how to create an AutoML video action recognition model from a Python script, and then do a batch prediction using the Vertex AI SDK.
-
-The steps performed include:
-
-- Create a `Vertex AI Dataset` resource.
-- Train the model.
-- View the model evaluation.
-- Make a batch prediction.
-```
-
-
-[AutoML Tabular Pipeline](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb)
-
-```
-Learn how to create two regression models using [Vertex Pipelines](https://cloud.
-
-The steps performed are:
-
-- Create a training pipeline that reduces the search space from the default to save time.
-- Create a training pipeline that reuses the architecture search results from the previous pipeline to save time.
-
-```
-
-
-[Training an AutoML text sentiment analysis model for online predictions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_text_sentiment_analysis_online.ipynb)
-
-```
-Learn how to create an AutoML text sentiment analysis model and deploy it for online predictions from a Python script using the Vertex AI SDK.
-
-The steps performed include:
-
-- Create a `Vertex AI Dataset` resource.
-- Create a training job for the AutoML model on the dataset.
-- View the model evaluation metrics.
-- Deploy the `Vertex AI Model` resource to a serving `Vertex AI Endpoint`.
-- Make a prediction request to the deployed model.
-- Undeploy the model from endpoint.
-- Perform clean up process.
-
-```
+   Learn more about [Classification for text data](https://cloud.google.com/vertex-ai/docs/training-overview#classification_for_text).
 
 
 [Compare Vertex AI Forecasting and BigQuery ML ARIMA_PLUS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb)
@@ -141,6 +54,114 @@ The steps performed are:
 
 ```
 
+   Learn more about [BQML ARIMA+ forecasting for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting-arima/overview).
+
+
+[AutoML Tabular Workflow pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb)
+
+```
+Learn how to create two regression models using [Vertex AI Pipelines](https://cloud.
+
+The steps performed are:
+
+- Create a training pipeline that reduces the search space from the default to save time.
+- Create a training pipeline that reuses the architecture search results from the previous pipeline to save time.
+
+```
+
+   Learn more about [Tabular Workflow for E2E AutoML](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/e2e-automl).
+
+
+[Get started with AutoML Training](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/get_started_automl_training.ipynb)
+
+```
+Learn how to use `AutoML` for training with `Vertex AI`.
+
+The steps performed include:
+
+- Train an image model
+- Export the image model as an edge model
+- Train a tabular model
+- Export the tabular model as a cloud model
+- Train a text model
+- Train a video model
+
+```
+
+   Learn more about [Vertex AI for AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users).
+
+
+[AutoML training hierarchical forecasting for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_forecasting_hierarchical_batch.ipynb)
+
+```
+In this tutorial, you create an AutoML hierarchical forecasting model and deploy it for batch prediction using the Vertex AI SDK for Python.
+
+The steps performed include:
+
+- Create a Vertex AI `TimeSeriesDataset` resource.
+- Train the model.
+- View the model evaluation.
+- Deploy the `Model` resource to a serving `Endpoint` resource.
+- Make a prediction.
+- Undeploy the `Model`.
+
+```
+
+   Learn more about [Hierarchical forecasting for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/hierarchical).
+
+
+[AutoML training image object detection model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb)
+
+```
+In this tutorial, you create an AutoML image object detection model from a Python script, and then do a batch prediction using the Vertex AI SDK.
+
+The steps performed include:
+
+- Create a Vertex `Dataset` resource.
+- Train the model.
+- View the model evaluation.
+- Make a batch prediction.
+
+```
+
+   Learn more about [Object detection for image data](https://cloud.google.com/vertex-ai/docs/training-overview#object_detection_for_images).
+
+
+[AutoML tabular forecasting model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_forecasting_batch.ipynb)
+
+```
+Learn how to create an `AutoML` tabular forecasting model from a Python script, and then do a batch prediction using the Vertex AI SDK.
+
+The steps performed include:
+
+- Create a `Vertex AI Dataset` resource.
+- Train an `AutoML` tabular forecasting `Model` resource.
+- Obtain the evaluation metrics for the `Model` resource.
+- Make a batch prediction.
+
+```
+
+   Learn more about [Forecasting for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/overview).
+
+
+[AutoML training tabular regression model for batch prediction using BigQuery](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_regression_batch_bq.ipynb)
+
+```
+Learn how to create an AutoML tabular regression model and deploy it for batch prediction using the Vertex AI SDK for Python.
+
+The steps performed include:
+
+- Create a Vertex AI `Dataset` resource.
+- Train the model.
+- View the model evaluation.
+- Deploy the `Model` resource to a serving `Endpoint` resource.
+- Make a prediction.
+- Undeploy the `Model`.
+
+```
+
+   Learn more about [Regression for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview).
+
 
 [AutoML training tabular regression model for online prediction using BigQuery](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb)
 
@@ -158,6 +179,81 @@ The steps performed include:
 
 ```
 
+   Learn more about [Regression for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview).
+
+
+[AutoML training text entity extraction model for online prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_text_entity_extraction_online.ipynb)
+
+```
+Learn how to create an AutoML text entity extraction model and deploy for online prediction from a Python script using the Vertex AI SDK.
+
+The steps performed include:
+
+- Create a Vertex `Dataset` resource.
+- Train the model.
+- View the model evaluation.
+- Deploy the `Model` resource to a serving `Endpoint` resource.
+- Make a prediction.
+- Undeploy the `Model`.
+
+```
+
+   Learn more about [Entity extraction for text data](https://cloud.google.com/vertex-ai/docs/training-overview#entity_extraction_for_text).
+
+
+[Training an AutoML text sentiment analysis model for online predictions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_text_sentiment_analysis_online.ipynb)
+
+```
+Learn how to create an AutoML text sentiment analysis model and deploy it for online predictions from a Python script using the Vertex AI SDK.
+
+The steps performed include:
+
+- Create a `Vertex AI Dataset` resource.
+- Create a training job for the AutoML model on the dataset.
+- View the model evaluation metrics.
+- Deploy the `Vertex AI Model` resource to a serving `Vertex AI Endpoint`.
+- Make a prediction request to the deployed model.
+- Undeploy the model from endpoint.
+- Perform clean up process.
+
+```
+
+   Learn more about [Sentiment analysis for text data](https://cloud.google.com/vertex-ai/docs/training-overview#sentiment_analysis_for_text).
+
+
+[AutoML training video action recognition model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_action_recognition_batch.ipynb)
+
+```
+Learn how to create an AutoML video action recognition model from a Python script, and then do a batch prediction using the Vertex AI SDK.
+
+The steps performed include:
+
+- Create a `Vertex AI Dataset` resource.
+- Train the model.
+- View the model evaluation.
+- Make a batch prediction.
+
+```
+
+   Learn more about [Action recognition for video data](https://cloud.google.com/vertex-ai/docs/training-overview#action_recognition_for_videos).
+
+
+[AutoML training video classification model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_classification_batch.ipynb)
+
+```
+Learn how to create an AutoML video classification model from a Python script, and then do a batch prediction using the Vertex AI SDK.
+
+The steps performed include:
+
+- Create a Vertex `Dataset` resource.
+- Train the model.
+- View the model evaluation.
+- Make a batch prediction.
+
+```
+
+   Learn more about [Classification for video data](https://cloud.google.com/vertex-ai/docs/training-overview#classification_for_videos).
+
 
 [AutoML training video object tracking model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_object_tracking_batch.ipynb)
 
@@ -170,21 +266,8 @@ The steps performed include:
 - Train the model.
 - View the model evaluation.
 - Make a batch prediction.
-```
-
-
-[AutoML training tabular regression model for batch prediction using BigQuery](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_regression_batch_bq.ipynb)
 
 ```
-Learn how to create an AutoML tabular regression model and deploy it for batch prediction using the Vertex AI SDK for Python.
 
-The steps performed include:
+   Learn more about [Object tracking for video data](https://cloud.google.com/vertex-ai/docs/training-overview#object_tracking_for_videos).
 
-- Create a Vertex AI `Dataset` resource.
-- Train the model.
-- View the model evaluation.
-- Deploy the `Model` resource to a serving `Endpoint` resource.
-- Make a prediction.
-- Undeploy the `Model`.
-
-```

notebooks/official/automl/automl-tabular-classification.ipynb

@@ -65,7 +65,7 @@
         "\n",
         "**Note**: you may incur charges for training, prediction, storage, or usage of other Google Cloud products in connection with testing this SDK.\n",
         "\n",
-        "Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/training-overview#tabular_data)."
+        "Learn more about [Classification for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview)."
       ]
     },
     {

notebooks/official/automl/automl-text-classification.ipynb

@@ -70,7 +70,7 @@
         "\n",
         "This notebook walks you through the major phases of building and using an AutoML text classification model on [Vertex AI](https://cloud.google.com/vertex-ai/docs/). \n",
         "\n",
-        "Learn more about [AutoML Text classification](https://cloud.google.com/vertex-ai/docs/text-data/classification/train-model)."
+        "Learn more about [Classification for text data](https://cloud.google.com/vertex-ai/docs/training-overview#classification_for_text)."
       ]
     },
     {

notebooks/official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "In this tutorial, you take on the role of a store planner who must determine how much inventory they will need to order for each of their products and stores for November 2019. You accomplish this by training forecasting models using historical sales data. You start with a baseline model using BigQuery ML (BQML) [ARIMA_PLUS](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-create-time-series) and then compare it against a [Vertex AI Forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/overview) model.\n",
         "\n",
-        "Learn more about [AutoML Forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/overview)  and [BQML Forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting-arima/overview)."
+        "Learn more about [BQML ARIMA+ forecasting for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting-arima/overview)."
       ]
     },
     {

notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "In this tutorial, you will use two Vertex AI Tabular Workflows pipelines to train AutoML models using different configurations. You will see how `get_automl_tabular_pipeline_and_parameters` gives you the ability to customize the default AutoML Tabular pipeline, and how `get_skip_architecture_search_pipeline_and_parameters` allows you to reduce the training time and cost for an AutoML model by using the tuning results from a previous pipeline run.\n",
         "\n",
-        "Learn more about [AutoML Tabular Workflows](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/e2e-automl)."
+        "Learn more about [Tabular Workflow for E2E AutoML](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/e2e-automl)."
       ]
     },
     {

notebooks/official/automl/sdk_automl_forecasting_hierarchical_batch.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK for Python to create hierarchical forecasting models using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users)and do batch prediction. Specifically, you predict a fictional store's sales based on historical sales data.\n",
         "\n",
-        "Learn more about [AutoML Forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/overview)."
+        "Learn more about [Hierarchical forecasting for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/hierarchical)."
       ]
     },
     {

notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb

@@ -65,7 +65,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK to create image object detection models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
         "\n",
-        "Learn more about [AutoML Image](https://cloud.google.com/vertex-ai/docs/image-data/object-detection/train-model)."
+        "Learn more about [Object detection for image data](https://cloud.google.com/vertex-ai/docs/training-overview#object_detection_for_images)."
       ]
     },
     {

notebooks/official/automl/sdk_automl_tabular_forecasting_batch.ipynb

@@ -64,7 +64,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK to create tabular forecasting models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
         "\n",
-        "Learn more about [AutoML Forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/tutorials-samples)."
+        "Learn more about [Forecasting for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/overview)."
       ]
     },
     {
@@ -833,6 +833,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "99b7a9287ba6"
@@ -840,7 +841,7 @@
       "source": [
         "For AutoML models, manual scaling can be adjusted by setting both min and max nodes i.e., `starting_replica_count` and `max_replica_count` as the same value(in this example, set to 1). The node count can be increased or decreased as required by load.\n",
         " \n",
-        "`batch_predict` can export predictions either to BigQuery or GCS. The BigQuery options are commented out below and the predictions will be exported to the BUCKET_URI."
+        "`batch_predict` can export predictions either to BigQuery or GCS. This example exports to BigQuery."
       ]
     },
     {

notebooks/official/automl/sdk_automl_tabular_regression_batch_bq.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK for Python to create tabular regression models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
         "\n",
-        "Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/training-overview#tabular_data)."
+        "Learn more about [Regression for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview)."
       ]
     },
     {

notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb

@@ -64,7 +64,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK for Python to create tabular regression models and do online prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
         "\n",
-        "Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/training-overview#tabular_data)."
+        "Learn more about [Regression for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview)."
       ]
     },
     {

notebooks/official/automl/sdk_automl_text_entity_extraction_online.ipynb

@@ -64,7 +64,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK for Python to create text entity extraction models and do online prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
         "\n",
-        "Learn more about [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/entity-extraction/train-model)."
+        "Learn more about [Entity extraction for text data](https://cloud.google.com/vertex-ai/docs/training-overview#entity_extraction_for_text)."
       ]
     },
     {

notebooks/official/automl/sdk_automl_text_sentiment_analysis_online.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK to train and deploy an [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) text sentiment analysis model and get online predictions from it.\n",
         "\n",
-        "Learn more about [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/sentiment-analysis/train-model)."
+        "Learn more about [Sentiment analysis for text data](https://cloud.google.com/vertex-ai/docs/training-overview#sentiment_analysis_for_text)."
       ]
     },
     {

notebooks/official/automl/sdk_automl_video_action_recognition_batch.ipynb

@@ -65,7 +65,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK to create video action recognition models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
         "\n",
-        "Learn more about [AutoML Video](https://cloud.google.com/vertex-ai/docs/video-data/action-recognition/train-model)."
+        "Learn more about [Action recognition for video data](https://cloud.google.com/vertex-ai/docs/training-overview#action_recognition_for_videos)."
       ]
     },
     {

notebooks/official/automl/sdk_automl_video_classification_batch.ipynb

@@ -64,7 +64,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK to create video classification models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
         "\n",
-        "Learn more about [AutoML Video](https://cloud.google.com/vertex-ai/docs/video-data/classification/train-model)."
+        "Learn more about [Classification for video data](https://cloud.google.com/vertex-ai/docs/training-overview#classification_for_videos)."
       ]
     },
     {

notebooks/official/automl/sdk_automl_video_object_tracking_batch.ipynb

@@ -64,7 +64,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK for Python to create video object tracking models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
         "\n",
-        "Learn more about [AutoML Video](https://cloud.google.com/vertex-ai/docs/video-data/object-tracking/train-model)."
+        "Learn more about [Object tracking for video data](https://cloud.google.com/vertex-ai/docs/training-overview#object_tracking_for_videos)."
       ]
     },
     {

notebooks/official/bigquery_ml/README.md

@@ -1,3 +1,4 @@
+
 [Online prediction with BigQuery ML](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/bigquery_ml/bqml-online-prediction.ipynb)
 
 ```
@@ -14,3 +15,25 @@ The steps performed include:
 
 ```
 
+   Learn more about [BigQuery ML](https://cloud.google.com/vertex-ai/docs/beginner/bqml).
+
+
+[Get started with BigQuery ML Training](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/bigquery_ml/get_started_with_bqml_training.ipynb)
+
+```
+Learn how to use `BigQueryML` for training with `Vertex AI`.
+
+The steps performed include:
+
+- Create a local BigQuery table in your project
+- Train a BigQuery ML model
+- Evaluate the BigQuery ML model
+- Export the BigQuery ML model as a cloud model
+- Upload the exported model as a `Vertex AI Model` resource
+- Hyperparameter tune a BigQuery ML model with `Vertex AI Vizier`
+- Automatically register a BigQuery ML model to `Vertex AI Model Registry`
+
+```
+
+   Learn more about [BigQuery ML](https://cloud.google.com/vertex-ai/docs/beginner/bqml).
+

notebooks/official/bigquery_ml/bqml-online-prediction.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "This notebook is aimed at data analysts and data scientists who have data in BigQuery, want to train a model using BigQuery ML, register the model to Vertex AI Model Registry, and deploy it to an endpoint for real-time prediction. \n",
         "\n",
-        "Learn more about [BigQuery ML](https://cloud.google.com/bigquery-ml/docs/introduction)."
+        "Learn more about [BigQuery ML](https://cloud.google.com/vertex-ai/docs/beginner/bqml)."
       ]
     },
     {

notebooks/official/custom/README.md

@@ -1,3 +1,24 @@
+
+[Deploying Iris-detection model using FastAPI and Vertex AI custom container serving](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/SDK_Custom_Container_Prediction.ipynb)
+
+```
+Learn how to create, deploy and serve a custom classification model on Vertex AI.
+
+The steps performed include:
+
+- Train a model that uses flower's measurements as input to predict the class of iris.
+- Save the model and its serialized pre-processor.
+- Build a FastAPI server to handle predictions and health checks.
+- Build a custom container with model artifacts.
+- Upload and deploy custom container to Vertex AI Endpoints.
+
+```
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+   Learn more about [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions).
+
+
 [Training and deploying a sales forecasting model using FBProphet and Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/SDK_FBProphet_Forecasting_Online.ipynb)
 
 ```
@@ -16,34 +37,9 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
 
-[Custom training and batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/sdk-custom-image-classification-batch.ipynb)
-
-```
-Learn to use `Vertex AI Training` to create a custom trained model and use `Vertex AI Batch Prediction` to do a batch prediction on the trained model.
-
-The steps performed include:
-
-- Create a `Vertex AI` custom job for training a TensorFlow model.
-- Upload the trained model artifacts as a `Model` resource.
-- Make a batch prediction.
-
-```
-
-
-[Profile model training performance using Profiler](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/custom_training_tensorboard_profiler.ipynb)
-
-```
-Learn how to enable Vertex AI TensorBoard Profiler for custom training jobs.
-
-The steps performed include:
-
-- Setup a service account and a Cloud Storage bucket
-- Create a TensorBoard instance
-- Create and run a custom training job
-- View the TensorBoard Profiler dashboard
-
-```
+   Learn more about [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions).
 
 
 [Training a TensorFlow model on BigQuery data](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb)
@@ -61,6 +57,43 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+
+[Profile model training performance using Profiler](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/custom_training_tensorboard_profiler.ipynb)
+
+```
+Learn how to enable Vertex AI TensorBoard Profiler for custom training jobs.
+
+The steps performed include:
+
+- Setup a service account and a Cloud Storage bucket
+- Create a TensorBoard instance
+- Create and run a custom training job
+- View the TensorBoard Profiler dashboard
+
+```
+
+   Learn more about [Vertex AI TensorBoard Profiler](https://cloud.google.com/vertex-ai/docs/experiments/tensorboard-profiler).
+
+
+[Custom training and batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/sdk-custom-image-classification-batch.ipynb)
+
+```
+Learn to use `Vertex AI Training` to create a custom trained model and use `Vertex AI Batch Prediction` to do a batch prediction on the trained model.
+
+The steps performed include:
+
+- Create a `Vertex AI` custom job for training a TensorFlow model.
+- Upload the trained model artifacts as a `Model` resource.
+- Make a batch prediction.
+
+```
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+   Learn more about [Vertex AI Batch Prediction](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/get-batch-predictions).
+
 
 [Custom training and online prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/sdk-custom-image-classification-online.ipynb)
 
@@ -78,19 +111,7 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
 
-[Deploying Iris-detection model using FastAPI and Vertex AI custom container serving](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/SDK_Custom_Container_Prediction.ipynb)
-
-```
-Learn how to create, deploy and serve a custom classification model on Vertex AI.
-
-The steps performed include:
-
-- Train a model that uses flower's measurements as input to predict the class of iris.
-- Save the model and its serialized pre-processor.
-- Build a FastAPI server to handle predictions and health checks.
-- Build a custom container with model artifacts.
-- Upload and deploy custom container to Vertex AI Endpoints.
-
-```
+   Learn more about [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions).
 

notebooks/official/datasets/README.md

@@ -0,0 +1,38 @@
+
+[Get started with BigQuery datasets](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/datasets/get_started_bq_datasets.ipynb)
+
+```
+Learn how to use `BigQuery` as a dataset for training with `Vertex AI`.
+
+The steps performed include:
+
+- Create a Vertex AI `Dataset` resource from `BigQuery` table -- compatible for `AutoML` training.
+- Extract a copy of the dataset from `BigQuery` to a CSV file in Cloud Storage -- compatible for `AutoML` or custom training.
+- Select rows from a `BigQuery` dataset into a `pandas` dataframe -- compatible for custom training.
+- Select rows from a `BigQuery` dataset into a `tf.data.Dataset` -- compatible for custom training `TensorFlow` models.
+- Select rows from extracted CSV files into a `tf.data.Dataset` -- compatible for custom training `TensorFlow` models.
+- Create a `BigQuery` dataset from CSV files.
+- Extract data from `BigQuery` table into a `DMatrix` -- compatible for custom training `XGBoost` models.
+
+```
+
+   Learn more about [BigQuery Datasets](https://cloud.google.com/bigquery/docs/datasets-intro).
+
+
+[Get started with Vertex AI Data Labeling](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/datasets/get_started_with_data_labeling.ipynb)
+
+```
+Learn how to use the `Vertex AI Data Labeling` service.
+
+The steps performed include:
+
+- Create a Specialist Pool for data labelers.
+- Create a data labeling job.
+- Submit the data labeling job.
+- List data labeling jobs.
+- Cancel a data labeling job.
+
+```
+
+   Learn more about [Vertex AI Data Labeling](https://cloud.google.com/vertex-ai/docs/datasets/data-labeling-job).
+

notebooks/official/experiments/README.md

@@ -1,17 +1,25 @@
-[Compare pipeline runs with Vertex AI Experiments](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/comparing_pipeline_runs.ipynb)
+
+[Build Vertex AI Experiment lineage for custom training](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb)
 
 ```
-Learn how to use `Vertex AI Experiments` to log a pipeline job and compare different pipeline jobs.
+Learn how to integrate preprocessing code in a Vertex AI experiments.
 
 The steps performed include:
 
-* Formalize a training component
-* Build a training pipeline
-* Run several Pipeline jobs and log their results
-* Compare different Pipeline jobs
+- Execute module for preprocessing data
+  - Create a dataset artifact
+  - Log parameters
+-  Execute module for training the model
+  - Log parameters
+  - Create model artifact
+  - Assign tracking lineage to dataset, model and parameters
 
 ```
 
+   Learn more about [Vertex AI Experiments](https://cloud.google.com/vertex-ai/docs/experiments/intro-vertex-ai-experiments).
+
+   Learn more about [Vertex ML Metadata](https://cloud.google.com/vertex-ai/docs/ml-metadata).
+
 
 [Track parameters and metrics for locally trained models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/comparing_local_trained_models.ipynb)
 
@@ -26,13 +34,57 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Experiments](https://cloud.google.com/vertex-ai/docs/experiments/intro-vertex-ai-experiments).
 
-[Build Vertex AI Experiment lineage for custom training](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb)
+
+[Compare pipeline runs with Vertex AI Experiments](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/comparing_pipeline_runs.ipynb)
 
 ```
-Learn how to integrate preprocessing code in a Vertex AI experiments.
+Learn how to use `Vertex AI Experiments` to log a pipeline job and compare different pipeline jobs.
 
+The steps performed include:
 
+* Formalize a training component
+* Build a training pipeline
+* Run several Pipeline jobs and log their results
+* Compare different Pipeline jobs
 
 ```
 
+   Learn more about [Vertex AI Experiments](https://cloud.google.com/vertex-ai/docs/experiments/intro-vertex-ai-experiments).
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+
+[Get started with Vertex AI Experiments](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/get_started_with_vertex_experiments.ipynb)
+
+```
+Learn how to use `Vertex AI Experiments` when training with `Vertex AI`.
+
+The steps performed include:
+
+- Local (notebook) Training
+    - Create an experiment
+    - Create a first run in the experiment
+    - Log parameters and metrics
+    - Create artifact lineage
+    - Visualize the experiment results
+    - Execute a second run
+    - Compare the two runs in the experiment
+- Cloud (`Vertex AI`) Training
+    - Within the training script:
+        - Create an experiment
+        - Log parameters and metrics
+        - Create artifact lineage
+    - Create a `Vertex AI Training` custom job
+    - Execute the custom job
+    - Visualize the experiment results
+
+```
+
+   Learn more about [Vertex AI Experiments](https://cloud.google.com/vertex-ai/docs/experiments/intro-vertex-ai-experiments).
+
+   Learn more about [Vertex ML Metadata](https://cloud.google.com/vertex-ai/docs/ml-metadata).
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+

notebooks/official/explainable_ai/README.md

@@ -1,18 +1,3 @@
-[Custom training tabular regression model for batch prediction with explainabilty](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb)
-
-```
-Learn how to use `Vertex AI Training and Explainable AI` to create a custom image classification model with explanations, and then you learn to use `Vertex AI Batch Prediction` to make a batch prediction request with explanations.
-
-The steps performed include:
-
-- Create a `Vertex AI` custom job for training a TensorFlow model.
-- View the model evaluation for the trained model.
-- Set explanation parameters for when the model is deployed.
-- Upload the trained model artifacts and explanations as a `Model` resource.
-- Make a batch prediction with explanations.
-
-```
-
 
 [AutoML training tabular binary classification model for batch explanation](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb)
 
@@ -28,6 +13,10 @@ The steps performed include:
 
 ```
 
+   Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/tabular-data/overview).
+
+   Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview).
+
 
 [AutoML training tabular classification model for online explanation](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb)
 
@@ -46,6 +35,30 @@ The steps performed include:
 
 ```
 
+   Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/tabular-data/overview).
+
+   Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview).
+
+
+[Custom training image classification model for batch prediction with explainabilty](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb)
+
+```
+Learn to use `Vertex AI Training and Explainable AI` to create a custom image classification model with explanations, and then you learn to use `Vertex AI Batch Prediction` to make a batch prediction request with explanations.
+
+The steps performed include:
+
+- Create a `Vertex AI` custom job for training a TensorFlow model.
+- View the model evaluation for the trained model.
+- Set explanation parameters for when the model is deployed.
+- Upload the trained model artifacts and explanation parameters as a `Model` resource.
+- Make a batch prediction with explanations.
+
+```
+
+   Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview).
+
+   Learn more about [Vertex AI Batch Prediction](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/get-batch-predictions).
+
 
 [Custom training image classification model for online prediction with explainabilty](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_image_classification_online_explain.ipynb)
 
@@ -65,6 +78,30 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview).
+
+   Learn more about [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions).
+
+
+[Custom training tabular regression model for batch prediction with explainabilty](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb)
+
+```
+Learn how to use `Vertex AI Training and Explainable AI` to create a custom image classification model with explanations, and then you learn to use `Vertex AI Batch Prediction` to make a batch prediction request with explanations.
+
+The steps performed include:
+
+- Create a `Vertex AI` custom job for training a TensorFlow model.
+- View the model evaluation for the trained model.
+- Set explanation parameters for when the model is deployed.
+- Upload the trained model artifacts and explanations as a `Model` resource.
+- Make a batch prediction with explanations.
+
+```
+
+   Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview).
+
+   Learn more about [Vertex AI Batch Prediction](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/get-batch-predictions).
+
 
 [Custom training tabular regression model for online prediction with explainabilty](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb)
 
@@ -84,11 +121,15 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview).
+
+   Learn more about [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions).
+
 
 [Custom training tabular regression model for online prediction with explainabilty using get_metadata](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb)
 
 ```
-Learn how to create a custom model from a Python script in a Google prebuilt Docker container using the Vertex SDK, and then do a prediction with explanations on the deployed model by sending data.
+Learn how to create a custom model from a Python script in a Google prebuilt Docker container using the Vertex AI SDK, and then do a prediction with explanations on the deployed model by sending data.
 
 The steps performed include:
 
@@ -104,19 +145,7 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview).
 
-[Custom training image classification model for batch prediction with explainabilty](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb)
-
-```
-Learn to use `Vertex AI Training and Explainable AI` to create a custom image classification model with explanations, and then you learn to use `Vertex AI Batch Prediction` to make a batch prediction request with explanations.
-
-The steps performed include:
-
-- Create a `Vertex AI` custom job for training a TensorFlow model.
-- View the model evaluation for the trained model.
-- Set explanation parameters for when the model is deployed.
-- Upload the trained model artifacts and explanation parameters as a `Model` resource.
-- Make a batch prediction with explanations.
-
-```
+   Learn more about [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions).
 

notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb

@@ -64,7 +64,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK to create tabular binary classification models and do batch prediction with explanation using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
         "\n",
-        "Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/tabular-data/overview) and [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview)."
+        "Learn more about [Classification for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview). Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview)."
       ]
     },
     {

notebooks/official/feature_store/README.md

@@ -1,3 +1,20 @@
+
+[Streaming ingestion SDK](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/feature_store_streaming_ingestion_sdk.ipynb)
+
+```
+Learn how to ingest features from a `Pandas DataFrame` into your Vertex AI Feature Store using `write_feature_values` method from the Vertex AI SDK.
+
+The steps performed include:
+
+- Create `Feature Store`
+- Create new `Entity Type` for your `Feature Store`
+- Ingest feature values from `Pandas DataFrame` into `Feature Store`'s `Entity Types`.
+
+```
+
+   Learn more about [Vertex AI Feature Store](https://cloud.google.com/vertex-ai/docs/featurestore).
+
+
 [Using Vertex AI Feature Store with Pandas Dataframe](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/sdk-feature-store-pandas.ipynb)
 
 ```
@@ -15,6 +32,8 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Feature Store](https://cloud.google.com/vertex-ai/docs/featurestore).
+
 
 [Online and Batch predictions using Vertex AI Feature Store](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/sdk-feature-store.ipynb)
 
@@ -27,6 +46,9 @@ The steps performed include:
 - Import feature data into `Vertex AI Feature Store` resource.
 - Serve online prediction requests using the imported features.
 - Access imported features in offline jobs, such as training jobs.
+- Use streaming ingestion to ingest small amount of data.
 
 ```
 
+   Learn more about [Vertex AI Feature Store](https://cloud.google.com/vertex-ai/docs/featurestore).
+

notebooks/official/matching_engine/README.md

@@ -1,18 +1,3 @@
-[Create Vertex AI Matching Engine index](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb)
-
-```
-Learn how to create Approximate Nearest Neighbor (ANN) Index, query against indexes, and validate the performance of the index.
-
-The steps performed include:
-
-* Create ANN Index and Brute Force Index
-* Create an IndexEndpoint with VPC Network
-* Deploy ANN Index and Brute Force Index
-* Perform online query
-* Compute recall
-
-```
-
 
 [Introduction to builtin Swivel embedding algorithm](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/intro-swivel.ipynb)
 
@@ -30,11 +15,31 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Matching Engine](https://cloud.google.com/vertex-ai/docs/matching-engine/overview).
 
-[Introduction to builtin Two-towers embedding algorithm](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/two-tower-model-introduction.ipynb)
+
+[Create Vertex AI Matching Engine index](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb)
 
 ```
-Learn how to run the two-tower model.
+Learn how to create Approximate Nearest Neighbor (ANN) Index, query against indexes, and validate the performance of the index.
+
+The steps performed include:
+
+* Create ANN Index and Brute Force Index
+* Create an IndexEndpoint with VPC Network
+* Deploy ANN Index and Brute Force Index
+* Perform online query
+* Compute recall
+
+```
+
+   Learn more about [Vertex AI Matching Engine](https://cloud.google.com/vertex-ai/docs/matching-engine/overview).
+
+
+[Introduction to builtin Two-Towers embedding algorithm](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/two-tower-model-introduction.ipynb)
+
+```
+Learn how to run the Two-Tower model.
 
 The steps performed include:
 1. **Setup**: Importing the required libraries and setting your global variables.
@@ -47,3 +52,5 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Matching Engine](https://cloud.google.com/vertex-ai/docs/matching-engine/overview).
+

notebooks/official/migration/README.md

@@ -1,3 +1,117 @@
+
+[AutoML Image Classification](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ1 Vertex SDK AutoML Image Classification.ipynb)
+
+```
+Learn to use `AutoML` to train an image model and use `Vertex AI Prediction` and `Vertex AI Batch Prediction` to do online and batch predictions.
+
+The steps performed include:
+
+- Train an AutoML image classification model.
+- Make a batch prediction.
+- Deploy model to a endpoint
+- Make a online prediction
+
+```
+
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
+
+   Learn more about [AutoML Image](https://cloud.google.com/vertex-ai/docs/tutorials/image-recognition-automl/training).
+
+
+[Custom Scikit-Learn model with pre-built training container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ10 Vertex SDK Custom Scikit-Learn with pre-built training container.ipynb)
+
+```
+Learn to use `Vertex AI Training` to create a custom trained model and use `Vertex AI Batch Prediction` to do a batch prediction on the trained model.
+
+The steps performed include:
+
+- Create a `Vertex AI` custom job for training a scikit-learn model.
+- Upload the trained model artifacts as a `Model` resource.
+- Make a batch prediction.
+- Deploy model to a endpoint
+- Make a online prediction
+
+```
+
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+
+[Hyperparameter Tuning](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ11 Vertex SDK Hyperparameter Tuning.ipynb)
+
+```
+Learn to use `Vertex AI Hyperparameter` to create and tune a custom trained model.
+
+The steps performed include:
+
+- Create a `Vertex AI` hyperparameter tuning job for training a TensorFlow model.
+
+```
+
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/training/hyperparameter-tuning-overview).
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+
+[AutoML Video Classificaton](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ14 Vertex SDK AutoML Video Classification.ipynb)
+
+```
+Learn to use `AutoML` to train a video model and use `Vertex AI Batch Prediction` to do batch predictions.
+
+The steps performed include:
+
+- Train an AutoML video classification model.
+- Make a batch prediction.
+
+```
+
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
+
+   Learn more about [AutoML Video](https://cloud.google.com/vertex-ai/docs/tutorials/video-classification-automl/training).
+
+
+[AutoML Video Object Tracking](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ15 Vertex SDK AutoML Object Tracking.ipynb)
+
+```
+Learn to use `AutoML` to train a video model and use `Vertex AI Batch Prediction` to do batch predictions.
+
+The steps performed include:
+
+- Train an AutoML video object tracking model.
+- Make a batch prediction.
+
+```
+
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
+
+   Learn more about [AutoML Video](https://cloud.google.com/video-intelligence/automl/object-tracking/docs/index-object-tracking).
+
+
+[Custom Image Classification w/pre-built training container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ2,12 Vertex SDK Custom Image Classification with pre-built training container.ipynb)
+
+```
+Learn how to train a tensorflow image classification model using a prebuilt container and Vertex AI training.
+
+The steps performed include:
+
+- *Package the training code into a python application.*
+- *Containerize the training application using Cloud Build and Artifact Registry.*
+- *Create a custom container training job in Vertex AI and run it.*
+- *Evaluate the model generated from the training job.*
+- *Create a model resource for the trained model in Vertex AI Model Registry.*
+- *Run a Vertex AI batch prediction job.*
+- *Deploy the model resource to a Vertex AI Endpoint.*
+- *Run a online prediction job on the model resource.*
+- *Clean up the resources created.*
+
+```
+
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+
 [Custom Image Classification w/custom training container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ3 Vertex SDK Custom Image Classification with custom training container.ipynb)
 
 ```
@@ -17,6 +131,10 @@ The steps performed include:
 
 ```
 
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
 
 [AutoML Tabular Binary Classification](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ4 Vertex SDK AutoML Tabular Binary Classification.ipynb)
 
@@ -34,39 +152,28 @@ The steps performed include:
 
 ```
 
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
 
-[Custom Scikit-Learn model with pre-built training container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ10 Vertex SDK Custom Scikit-Learn with pre-built training container.ipynb)
+   Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/start/automl-users#tables).
+
+
+[AutoML Image Object Detection](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ5 Vertex SDK AutoML Image Object Detection.ipynb)
 
 ```
-Learn to use `Vertex AI Training` to create a custom trained model and use `Vertex AI Batch Prediction` to do a batch prediction on the trained model.
+Learn to use `AutoML` to train an image model and use `Vertex AI Prediction` and `Vertex AI Batch Prediction` to do online and batch predictions.
 
 The steps performed include:
 
-- Create a `Vertex AI` custom job for training a scikit-learn model.
-- Upload the trained model artifacts as a `Model` resource.
+- Train an AutoML object detection model.
 - Make a batch prediction.
 - Deploy model to a endpoint
 - Make a online prediction
 
 ```
 
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
 
-[AutoML Text Entity Extraction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ7 Vertex SDK AutoML Text Entity Extraction.ipynb)
-
-```
-The objective of this notebook is to build a AutoML Text Entity Extrasction Model.
-
-The steps performed include the following:
-
-* Set your task name, and GCS prefix
-* Copy AutoML video demo train data for creating managed dataset
-* Create a dataset on Vertex AI.
-* Configure a training job
-* Launch a training job and create a model on Vertex AI
-* Copy AutoML Video Demo Prediction Data for creating batch prediction job
-* Perform batch prediction job on the model
-
-```
+   Learn more about [AutoML Image](https://cloud.google.com/vertex-ai/docs/tutorials/image-recognition-automl/training).
 
 
 [AutoML Text Classification](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ6 Vertex SDK AutoML Text Classification.ipynb)
@@ -86,3 +193,70 @@ The steps performed include the following:
 
 ```
 
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
+
+   Learn more about [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/classification/prepare-data).
+
+
+[AutoML Text Entity Extraction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ7 Vertex SDK AutoML Text Entity Extraction.ipynb)
+
+```
+The objective of this notebook is to build a AutoML Text Entity Extraction Model.
+
+The steps performed include the following:
+
+* Set your task name, and GCS prefix
+* Copy AutoML video demo train data for creating managed dataset
+* Create a dataset on Vertex AI.
+* Configure a training job
+* Launch a training job and create a model on Vertex AI
+* Copy AutoML Video Demo Prediction Data for creating batch prediction job
+* Perform batch prediction job on the model
+
+```
+
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
+
+   Learn more about [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/entity-extraction/prepare-data).
+
+
+[AutoML Text Sentiment Analysis](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ8 Vertex SDK AutoML Text Sentiment Analysis.ipynb)
+
+```
+The objective of this notebook is to build a AutoML Text Sentiment Analysis model.
+
+The steps performed include the following:
+
+* Copy AutoML video demo train data for creating managed dataset
+* Create a dataset on Vertex AI.
+* Configure a training job
+* Launch a training job and create a model on Vertex AI
+* Copy AutoML Video Demo Prediction Data for creating batch prediction job
+* Perform batch prediction job on the model
+
+```
+
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
+
+   Learn more about [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/sentiment-analysis/prepare-data).
+
+
+[Custom XGBoost model with pre-built training container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ9 Vertex SDK Custom XGBoost with pre-built training container.ipynb)
+
+```
+Learn to use `Vertex AI Training` to create a custom trained model and use `Vertex AI Batch Prediction` to do a batch prediction on the trained model.
+
+The steps performed include:
+
+- Create a `Vertex AI` custom job for training a scikit-learn model.
+- Upload the trained model artifacts as a `Model` resource.
+- Make a batch prediction.
+- Deploy model to a endpoint
+- Make a online prediction
+
+```
+
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+

notebooks/official/migration/UJ4 Vertex SDK AutoML Tabular Binary Classification.ipynb

@@ -64,7 +64,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK to create tabular binary classification models and do online prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
         "\n",
-        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/start/automl-users#tables)."
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai). Learn more about [Classification for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview)."
       ]
     },
     {

notebooks/official/ml_metadata/README.md

@@ -1,3 +1,20 @@
+
+[Track parameters and metrics for custom training jobs](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb)
+
+```
+Learn how to use Vertex AI SDK for Python to:
+
+The steps performed include:
+- Track training parameters and prediction metrics for a custom training job.
+- Extract and perform analysis for all parameters and metrics within an Experiment.
+
+```
+
+   Learn more about [Vertex ML Metadata](https://cloud.google.com/vertex-ai/docs/ml-metadata).
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+
 [Track parameters and metrics for locally trained models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb)
 
 ```
@@ -10,17 +27,7 @@ The steps performed include:
 
 ```
 
-
-[Track parameters and metrics for custom training jobs](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb)
-
-```
-Learn how to use Vertex AI SDK for Python to:
-
-The steps performed include:
-- Track training parameters and prediction metrics for a custom training job.
-- Extract and perform analysis for all parameters and metrics within an Experiment.
-
-```
+   Learn more about [Vertex ML Metadata](https://cloud.google.com/vertex-ai/docs/ml-metadata).
 
 
 [Track artifacts and metrics across Vertex AI Pipelines runs using Vertex ML Metadata](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/ml_metadata/vertex-pipelines-ml-metadata.ipynb)
@@ -39,3 +46,7 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex ML Metadata](https://cloud.google.com/vertex-ai/docs/ml-metadata).
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+

notebooks/official/model_evaluation/README.md

@@ -1,3 +1,4 @@
+
 [Evaluating batch prediction results from an AutoML Tabular classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/automl_tabular_classification_model_evaluation.ipynb)
 
 ```
@@ -14,22 +15,9 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction).
 
-[Evaluating batch prediction results from AutoML Video classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/automl_video_classification_model_evaluation.ipynb)
-
-```
-Learn how to train a Vertex AI AutoML Video classification model and learn how to evaluate it through a Vertex AI pipeline job using `google_cloud_pipeline_components`:
-
-The steps performed include:
-
-- Create a `Vertex AI Dataset`.
-- Train a Automl Video Classification model on the `Vertex AI Dataset` resource.
-- Import the trained `AutoML Vertex AI Model resource` into the pipeline.
-- Run a batch prediction job inside the pipeline.
-- Evaulate the AutoML model using the classification evaluation component.
-- Import the classification metrics to the AutoML Vertex AI Model resource.
-
-```
+   Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/start/automl-users#tables).
 
 
 [Evaluating batch prediction results from AutoML Tabular regression model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/automl_tabular_regression_model_evaluation.ipynb)
@@ -49,25 +37,9 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction).
 
-[Evaluating batch prediction results from custom tabular regression model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/custom_tabular_regression_model_evaluation.ipynb)
-
-```
-Learn how to evaluate a Vertex AI model resource through a Vertex AI pipeline job using `google_cloud_pipeline_components`:
-
-The steps performed include:
-
-- Create a Vertex AI `CustomTrainingJob` for training a model.
-- Run the `CustomTrainingJob` 
-- Retrieve and load the model artifacts.
-- View the model evaluation.
-- Upload the model as a Vertex AI Model resource.
-- Import a pre-trained `Vertex AI model resource` into the pipeline.
-- Run a `batch prediction` job in the pipeline.
-- Evaulate the model using the `regression evaluation component`.
-- Import the Regression Metrics to the Vertex AI model resource.
-
-```
+   Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/start/automl-users#tables).
 
 
 [AutoML text classification pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/automl_text_classification_model_evaluation.ipynb)
@@ -86,6 +58,31 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction).
+
+   Learn more about [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/classification/prepare-data).
+
+
+[Evaluating batch prediction results from AutoML Video classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/automl_video_classification_model_evaluation.ipynb)
+
+```
+Learn how to train a Vertex AI AutoML Video classification model and learn how to evaluate it through a Vertex AI pipeline job using `google_cloud_pipeline_components`:
+
+The steps performed include:
+
+- Create a `Vertex AI Dataset`.
+- Train a Automl Video Classification model on the `Vertex AI Dataset` resource.
+- Import the trained `AutoML Vertex AI Model resource` into the pipeline.
+- Run a batch prediction job inside the pipeline.
+- Evaulate the AutoML model using the classification evaluation component.
+- Import the classification metrics to the AutoML Vertex AI Model resource.
+
+```
+
+   Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction).
+
+   Learn more about [AutoML Video](https://cloud.google.com/vertex-ai/docs/video-data/classification/prepare-data).
+
 
 [Evaluating BatchPrediction results from a Custom Tabular classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/custom_tabular_classification_model_evaluation.ipynb)
 
@@ -109,3 +106,31 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction).
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+
+[Evaluating batch prediction results from custom tabular regression model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/custom_tabular_regression_model_evaluation.ipynb)
+
+```
+Learn how to evaluate a Vertex AI model resource through a Vertex AI pipeline job using `google_cloud_pipeline_components`:
+
+The steps performed include:
+
+- Create a Vertex AI `CustomTrainingJob` for training a model.
+- Run the `CustomTrainingJob` 
+- Retrieve and load the model artifacts.
+- View the model evaluation.
+- Upload the model as a Vertex AI Model resource.
+- Import a pre-trained `Vertex AI model resource` into the pipeline.
+- Run a `batch prediction` job in the pipeline.
+- Evaulate the model using the `regression evaluation component`.
+- Import the Regression Metrics to the Vertex AI model resource.
+
+```
+
+   Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction).
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+

notebooks/official/model_evaluation/automl_tabular_classification_model_evaluation.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "This notebook demonstrates how to use the Vertex AI classification model evaluation component to evaluate an AutoML Tabular classification model. Model evaluation helps determine your model's performance based on the evaluation metrics and improve the model whenever necessary. \n",
         "\n",
-        "Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction) and [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/start/automl-users#tables)."
+        "Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction). Learn more about [Classification for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview)."
       ]
     },
     {

notebooks/official/model_evaluation/automl_tabular_regression_model_evaluation.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "This notebook demonstrates how to use Vertex AI regression model evaluation component to evaluate an AutoML Tabular regression model. Model evaluation helps you determine your model performance based on the evaluation metrics and improve the model if necessary. \n",
         "\n",
-        "Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction) and [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/start/automl-users#tables)."
+        "Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction). Learn more about [Regression for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview)."
       ]
     },
     {

notebooks/official/model_monitoring/README.md

@@ -1,7 +1,4 @@
 
-### model_monitoring 
-
-
 [Vertex AI Batch Prediction with Model Monitoring](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/batch_prediction_model_monitoring.ipynb)
 
 ```
@@ -15,6 +12,130 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Model Monitoring for batch predictions](https://cloud.google.com/vertex-ai/docs/model-monitoring/model-monitoring-batch-predictions).
+
+
+[Vertex AI Model Monitoring for AutoML tabular models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/get_started_with_model_monitoring_automl.ipynb)
+
+```
+Learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests, for AutoML tabular models.
+
+The steps performed include:
+
+- Train an `AutoML` model.
+- Deploy the `Model` resource to the `Endpoint` resource.
+- Configure the `Endpoint` resource for model monitoring.
+- Generate synthetic prediction requests for skew.
+- Wait for email alert notification.
+- Generate synthetic prediction requests for drift.
+- Wait for email alert notification.
+
+```
+
+   Learn more about [Vertex AI Model Monitoring](https://cloud.google.com/vertex-ai/docs/model-monitoring).
+
+
+[Vertex AI Model Monitoring for batch prediction in AutoML image models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/get_started_with_model_monitoring_automl_image_batch.ipynb)
+
+```
+Learn how to use `Vertex AI Model Monitoring` with `Vertex AI Batch Prediction` with an AutoML image classification model to detect an out of distribution image.
+
+The steps performed include:
+
+1. Train an AutoML image classification model.
+2. Submit a batch prediction containing both in and out of distribution images.
+3. Use Model Monitoring to calculate anomaly score on each image.
+4. Identify the images in the batch prediction request that are out of distribution.
+
+```
+
+   Learn more about [Vertex AI Model Monitoring](https://cloud.google.com/vertex-ai/docs/model-monitoring).
+
+
+[Vertex AI Model Monitoring for custom tabular models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/get_started_with_model_monitoring_custom.ipynb)
+
+```
+Learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests, for custom tabular models.
+
+The steps performed include:
+
+- Download a pre-trained custom tabular model.
+- Upload the pre-trained model as a `Model` resource.
+- Deploy the `Model` resource to the `Endpoint` resource.
+- Configure the `Endpoint` resource for model monitoring.
+- Generate synthetic prediction requests for skew.
+- Wait for email alert notification.
+- Generate synthetic prediction requests for drift.
+- Wait for email alert notification.
+
+```
+
+   Learn more about [Vertex AI Model Monitoring](https://cloud.google.com/vertex-ai/docs/model-monitoring).
+
+
+[Vertex AI Model Monitoring for custom tabular models with TensorFlow Serving container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/get_started_with_model_monitoring_custom_tf_serving.ipynb)
+
+```
+Learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests, for custom tabular models, using a custom deployment container.
+
+The steps performed include:
+
+- Download a pre-trained custom tabular model.
+- Upload the pre-trained model as a `Model` resource.
+- Deploying the `Model` resource to an `Endpoint` resource with `TensorFlow Serving` serving binary.
+- Configure the `Endpoint` resource for model monitoring.
+- Generate synthetic prediction requests for skew.
+- Wait for email alert notification.
+- Generate synthetic prediction requests for drift.
+- Wait for email alert notification.
+
+```
+
+   Learn more about [Vertex AI Model Monitoring](https://cloud.google.com/vertex-ai/docs/model-monitoring).
+
+
+[Vertex AI Model Monitoring for setup for tabular models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/get_started_with_model_monitoring_setup.ipynb)
+
+```
+Learn to setup the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests.
+
+The steps performed include:
+
+- Download a pre-trained custom tabular model.
+- Upload the pre-trained model as a `Model` resource.
+- Deploy the `Model` resource to the `Endpoint` resource.
+- Configure the `Endpoint` resource for model monitoring.
+    - Skew and drift detection for feature inputs.
+    - Skew and drift detection for feature attributions.
+- Automatic generation of the `input schema` by sending 1000 prediction request.
+- List, pause, resume and delete monitoring jobs.
+- Restart monitoring job with predefined `input schema`.
+- View logged monitored data.
+
+```
+
+   Learn more about [Vertex AI Model Monitoring](https://cloud.google.com/vertex-ai/docs/model-monitoring).
+
+
+[Vertex AI Model Monitoring for XGBoost models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/get_started_with_model_monitoring_xgboost.ipynb)
+
+```
+Learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests for XGBoost models.
+
+The steps performed include:
+
+- Download a pre-trained XGBoost model.
+- Upload the pre-trained model as a `Model` resource.
+- Deploy the `Model` resource to the `Endpoint` resource.
+- Configure the `Endpoint` resource for model monitoring:
+  - drift detection only -- no access to training data.
+  - predefine the input schema to map feature alias names to the unnamed array input to the model.
+- Generate synthetic prediction requests for drift.
+
+```
+
+   Learn more about [Vertex AI Model Monitoring](https://cloud.google.com/vertex-ai/docs/model-monitoring).
+
 
 [Vertex AI Model Monitoring with Explainable AI Feature Attributions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/model_monitoring.ipynb)
 
@@ -33,3 +154,5 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Model Monitoring](https://cloud.google.com/vertex-ai/docs/model-monitoring).
+

notebooks/official/model_monitoring/batch_prediction_model_monitoring.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "In this notebook, you will learn how to use Model Monitoring with batch prediction requests on a deployed Vertex AI Model resource. In a companion notebook, <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/model_monitoring.ipynb\" target=\"_blank\">Vertex AI Model Monitoring with Explainable AI Feature Attributions</a>, you can learn about how to apply model monitoring to streaming, real-time predictions."
+        "In this notebook, you will learn how to use Model Monitoring with batch prediction requests on a deployed Vertex AI Model resource. In a companion notebook, <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/model_monitoring.ipynb\" target=\"_blank\">Vertex AI Model Monitoring with Explainable AI Feature Attributions</a>, you can learn about how to apply model monitoring to streaming, real-time predictions.\n",
+        "\n",
+        "Learn more about [Vertex AI Model Monitoring for batch predictions](https://cloud.google.com/vertex-ai/docs/model-monitoring/model-monitoring-batch-predictions)."
       ]
     },
     {

notebooks/official/model_monitoring/model_monitoring.ipynb

@@ -80,7 +80,9 @@
         "\n",
         "[Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview) adds another facet to model monitoring, which we call feature attribution monitoring. Explainable AI enables you to understand the relative contribution of each feature to a resulting prediction. In essence, it assesses the magnitude of each feature's influence.\n",
         "\n",
-        "If production traffic differs from  training data, or varies substantially over time, **either in terms of model predictions or feature attributions**, that's likely to impact the quality of the answers your model produces. When that happens, you'd like to be alerted automatically and responsively, so that **you can anticipate problems before they affect your customer experiences or your revenue streams**."
+        "If production traffic differs from  training data, or varies substantially over time, **either in terms of model predictions or feature attributions**, that's likely to impact the quality of the answers your model produces. When that happens, you'd like to be alerted automatically and responsively, so that **you can anticipate problems before they affect your customer experiences or your revenue streams**.\n",
+        "\n",
+        "Learn more about [Vertex AI Model Monitoring](https://cloud.google.com/vertex-ai/docs/model-monitoring)."
       ]
     },
     {

notebooks/official/model_registry/README.md

@@ -1,7 +1,4 @@
 
-### model_registry 
-
-
 [Deploy BiqQuery ML Model on Vertex AI Model Registry and make predictions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_registry/bqml_vertexai_model_registry.ipynb)
 
 ```
@@ -18,3 +15,7 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).
+
+   Learn more about [BigQuery ML](https://cloud.google.com/vertex-ai/docs/beginner/bqml).
+

notebooks/official/pipelines/README.md

@@ -1,76 +1,4 @@
 
-### pipelines 
-
-
-[AutoML image classification pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_images.ipynb)
-
-```
-Learn how to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` image classification model.
-
-The steps performed include:
-
-- Create a KFP pipeline:
-    - Create a `Dataset` resource.
-    - Train an AutoML image classification `Model` resource.
-    - Create an `Endpoint` resource.
-    - Deploys the `Model` resource to the `Endpoint` resource.
-- Compile the KFP pipeline.
-- Execute the KFP pipeline using `Vertex AI Pipelines`
-
-```
-
-
-[Metrics visualization and run comparison using the KFP SDK](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb)
-
-```
-Learn how to use the KFP SDK to build pipelines that generate evaluation metrics.
-
-The steps performed include:
-
-- Create KFP components:
-    - Generate ROC curve and confusion matrix visualizations for classification results
-    - Write metrics
-- Create KFP pipelines.
-- Execute KFP pipelines
-- Compare metrics across pipeline runs
-
-```
-
-
-[Lightweight Python function-based components, and component I/O](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb)
-
-```
-Learn to use the KFP SDK to build lightweight Python function-based components, and then you learn to use `Vertex AI Pipelines` to execute the pipeline.
-
-The steps performed include:
-
-- Build Python function-based KFP components.
-- Construct a KFP pipeline.
-- Pass *Artifacts* and *parameters* between components, both by path reference and by value.
-- Use the `kfp.dsl.importer` method.
-- Compile the KFP pipeline.
-- Execute the KFP pipeline using `Vertex AI Pipelines`
-
-```
-
-
-[Custom training with pre-built Google Cloud Pipeline Components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb)
-
-```
-Learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build a custom model.
-
-The steps performed include:
-
-- Create a KFP pipeline:
-    - Train a custom model.
-    - Upload the trained model as a `Model` resource.
-    - Create an `Endpoint` resource.
-    - Deploy the `Model` resource to the `Endpoint` resource.
-    - Make a batch prediction request.
-
-```
-
-
 [AutoML Tabular pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/automl_tabular_classification_beans.ipynb)
 
 ```
@@ -88,68 +16,9 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
 
-[Training an acquisition-prediction model using Swivel, BigQuery ML and Vertex AI Pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb)
-
-```
-Learn how to build a simple BigQuery ML pipeline using Vertex AI pipelines in order to calculate text embeddings of content from articles and classify them
-into the *corporate acquisitions* category.
-
-The steps performed include:
-
-- Creating a component for Dataflow job that ingests data to BigQuery.
-- Creating a component for preprocessing steps to run on the data in BigQuery.
-- Creating a component for training a logistic regression model using BigQuery ML.
-- Building and configuring a Kubeflow DSL pipeline with all the created components.
-- Compiling and running the pipeline in Vertex AI Pipelines.
-
-```
-
-
-[Pipelines introduction for KFP](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/pipelines_intro_kfp.ipynb)
-
-```
-Learn how to use the KFP SDK to build pipelines that generate evaluation metrics.
-
-The steps performed include:
-
-- Define and compile a `Vertex AI` pipeline.
-- Specify which service account to use for a pipeline run.
-
-```
-
-
-[AutoML tabular regression pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb)
-
-```
-Learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` tabular regression model.
-
-The steps performed include:
-
-- Create a KFP pipeline:
-    - Create a `Dataset` resource.
-    - Train an AutoML tabular regression `Model` resource.
-    - Create an `Endpoint` resource.
-    - Deploys the `Model` resource to the `Endpoint` resource.
-- Compile the KFP pipeline.
-- Execute the KFP pipeline using `Vertex AI Pipelines`
-
-```
-
-
-[Model upload, predict, and evaluate using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_model_upload_predict_evaluate.ipynb)
-
-```
-Learn how to evaluate a custom model using a pipeline with components from `google_cloud_pipeline_components` and a custom pipeline component you build.
-
-The steps performed include:
-
-- Upload a pre-trained model as a `Model` resource.
-- Run a `BatchPredictionJob` on the `Model` resource with ground truth data.
-- Generate evaluation `Metrics` artifact about the `Model` resource.
-- Compare the evaluation metrics to a threshold.
-
-```
+   Learn more about [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component).
 
 
 [Pipeline control structures using the KFP SDK](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/control_flow_kfp.ipynb)
@@ -166,58 +35,29 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
 
-[Loan eligibility prediction using `google-cloud-pipeline-components` and Spark ML](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_dataproc_tabular.ipynb)
+
+[Custom training with pre-built Google Cloud Pipeline Components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb)
 
 ```
-Learn how to build a Vertex AI pipeline and train a Random-forest model using Spark ML for loan-eligibility classification problem.
-
-The steps performed include:
-
-*   Use the `DataprocPySparkBatchOp` to preprocess data.
-*   Create a Vertex AI dataset resource on the training data.
-*   Train a random forest model using Pyspark.
-*   Build a Vertex AI pipeline and run the training job.
-*   Use the Spark serving image in order to deploy a Spark model on Vertex AI Endpoint.
-
-```
-
-
-[Model train, upload, and deploy using Google Cloud Pipeline Components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb)
-
-```
-Learn how to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build and deploy a custom model.
+Learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build a custom model.
 
 The steps performed include:
 
 - Create a KFP pipeline:
     - Train a custom model.
-    - Uploads the trained model as a `Model` resource.
-    - Creates an `Endpoint` resource.
-    - Deploys the `Model` resource to the `Endpoint` resource.
-- Compile the KFP pipeline.
-- Execute the KFP pipeline using `Vertex AI Pipelines`
-
-```
-
-
-[AutoML text classification pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_text.ipynb)
-
-```
-Learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` text classification model.
-
-The steps performed include:
-
-- Create a KFP pipeline:
-    - Create a `Dataset` resource.
-    - Train an AutoML text classification `Model` resource.
+    - Upload the trained model as a `Model` resource.
     - Create an `Endpoint` resource.
-    - Deploys the `Model` resource to the `Endpoint` resource.
-- Compile the KFP pipeline.
-- Execute the KFP pipeline using `Vertex AI Pipelines`
+    - Deploy the `Model` resource to the `Endpoint` resource.
+    - Make a batch prediction request.
 
 ```
 
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+   Learn more about [Vertex AI Training components](https://cloud.google.com/vertex-ai/docs/training/create-training-pipeline).
+
 
 [Training and batch prediction with BigQuery source and destinantion for a custom tabular classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/custom_tabular_train_batch_pred_bq_pipeline.ipynb)
 
@@ -240,3 +80,229 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+   Learn more about [Vertex AI Batch Prediction components](https://cloud.google.com/vertex-ai/docs/pipelines/batchprediction-component).
+
+
+[AutoML image classification pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_images.ipynb)
+
+```
+Learn how to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` image classification model.
+
+The steps performed include:
+
+- Create a KFP pipeline:
+    - Create a `Dataset` resource.
+    - Train an AutoML image classification `Model` resource.
+    - Create an `Endpoint` resource.
+    - Deploys the `Model` resource to the `Endpoint` resource.
+- Compile the KFP pipeline.
+- Execute the KFP pipeline using `Vertex AI Pipelines`
+
+```
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+   Learn more about [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component).
+
+
+[AutoML tabular regression pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb)
+
+```
+Learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` tabular regression model.
+
+The steps performed include:
+
+- Create a KFP pipeline:
+    - Create a `Dataset` resource.
+    - Train an AutoML tabular regression `Model` resource.
+    - Create an `Endpoint` resource.
+    - Deploys the `Model` resource to the `Endpoint` resource.
+- Compile the KFP pipeline.
+- Execute the KFP pipeline using `Vertex AI Pipelines`
+
+```
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+   Learn more about [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component).
+
+
+[AutoML text classification pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_text.ipynb)
+
+```
+Learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` text classification model.
+
+The steps performed include:
+
+- Create a KFP pipeline:
+    - Create a `Dataset` resource.
+    - Train an AutoML text classification `Model` resource.
+    - Create an `Endpoint` resource.
+    - Deploys the `Model` resource to the `Endpoint` resource.
+- Compile the KFP pipeline.
+- Execute the KFP pipeline using `Vertex AI Pipelines`
+
+```
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+   Learn more about [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component).
+
+
+[Training an acquisition-prediction model using Swivel, BigQuery ML and Vertex AI Pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb)
+
+```
+Learn how to build a simple BigQuery ML pipeline using Vertex AI pipelines in order to calculate text embeddings of content from articles and classify them
+into the *corporate acquisitions* category.
+
+The steps performed include:
+
+- Creating a component for Dataflow job that ingests data to BigQuery.
+- Creating a component for preprocessing steps to run on the data in BigQuery.
+- Creating a component for training a logistic regression model using BigQuery ML.
+- Building and configuring a Kubeflow DSL pipeline with all the created components.
+- Compiling and running the pipeline in Vertex AI Pipelines.
+
+```
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+   Learn more about [BigQuery ML components](https://cloud.google.com/vertex-ai/docs/pipelines/bigqueryml-component).
+
+
+[Loan eligibility prediction using `google-cloud-pipeline-components` and Spark ML](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_dataproc_tabular.ipynb)
+
+```
+Learn how to build a Vertex AI pipeline and train a random-forest model using Spark ML for loan-eligibility classification problem.
+
+The steps performed include:
+
+*   Use the `DataprocPySparkBatchOp` to preprocess data.
+*   Create a Vertex AI dataset resource on the training data.
+*   Train a random forest model using PySpark.
+*   Build a Vertex AI pipeline and run the training job.
+*   Use the Spark serving image in order to deploy a Spark model on Vertex AI Endpoint.
+
+```
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+   Learn more about [Dataproc components](https://cloud.google.com/vertex-ai/docs/pipelines/dataproc-component).
+
+
+[Model train, upload, and deploy using Google Cloud Pipeline Components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb)
+
+```
+Learn how to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build and deploy a custom model.
+
+The steps performed include:
+
+- Create a KFP pipeline:
+    - Train a custom model.
+    - Uploads the trained model as a `Model` resource.
+    - Creates an `Endpoint` resource.
+    - Deploys the `Model` resource to the `Endpoint` resource.
+- Compile the KFP pipeline.
+- Execute the KFP pipeline using `Vertex AI Pipelines`
+
+```
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+   Learn more about [Vertex AI Training components](https://cloud.google.com/vertex-ai/docs/pipelines/customjob-component).
+
+
+[Model upload, predict, and evaluate using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_model_upload_predict_evaluate.ipynb)
+
+```
+Learn how to evaluate a custom model using a pipeline with components from `google_cloud_pipeline_components` and a custom pipeline component you build.
+
+The steps performed include:
+
+- Upload a pre-trained model as a `Model` resource.
+- Run a `BatchPredictionJob` on the `Model` resource with ground truth data.
+- Generate evaluation `Metrics` artifact about the `Model` resource.
+- Compare the evaluation metrics to a threshold.
+
+```
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+   Learn more about [Vertex AI Model components](https://cloud.google.com/vertex-ai/docs/pipelines/model-endpoint-component).
+
+
+[Lightweight Python function-based components, and component I/O](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb)
+
+```
+Learn to use the KFP SDK to build lightweight Python function-based components, and then you learn to use `Vertex AI Pipelines` to execute the pipeline.
+
+The steps performed include:
+
+- Build Python function-based KFP components.
+- Construct a KFP pipeline.
+- Pass *Artifacts* and *parameters* between components, both by path reference and by value.
+- Use the `kfp.dsl.importer` method.
+- Compile the KFP pipeline.
+- Execute the KFP pipeline using `Vertex AI Pipelines`
+
+```
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+
+[Metrics visualization and run comparison using the KFP SDK](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb)
+
+```
+Learn how to use the KFP SDK to build pipelines that generate evaluation metrics.
+
+The steps performed include:
+
+- Create KFP components:
+    - Generate ROC curve and confusion matrix visualizations for classification results
+    - Write metrics
+- Create KFP pipelines.
+- Execute KFP pipelines
+- Compare metrics across pipeline runs
+
+```
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+
+[Pipelines introduction for KFP](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/pipelines_intro_kfp.ipynb)
+
+```
+Learn how to use the KFP SDK to build pipelines that generate evaluation metrics.
+
+The steps performed include:
+
+- Define and compile a `Vertex AI` pipeline.
+- Specify which service account to use for a pipeline run.
+
+```
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+
+[BQML and AutoML - Experimenting with Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/rapid_prototyping_bqml_automl.ipynb)
+
+```
+Learn how to use `Vertex AI Predictions` for rapid prototyping a model.
+
+The steps performed include:
+
+- Creating a BigQuery and Vertex AI training dataset.
+- Training a BigQuery ML and AutoML model.
+- Extracting evaluation metrics from the BigQueryML and AutoML models.
+- Selecting the best trained model.
+- Deploying the best trained model.
+- Testing the deployed model infrastructure.
+
+```
+
+   Learn more about [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component).
+
+   Learn more about [BigQuery ML components](https://cloud.google.com/vertex-ai/docs/pipelines/bigqueryml-component).
+

notebooks/official/pipelines/automl_tabular_classification_beans.ipynb

@@ -68,7 +68,7 @@
         "\n",
         "<a href=\"https://storage.googleapis.com/amy-jo/images/mp/beans.png\" target=\"_blank\"><img src=\"https://storage.googleapis.com/amy-jo/images/mp/beans.png\" width=\"95%\"/></a>\n",
         "\n",
-        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component)."
+        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component). Learn more about [Classification for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview)."
       ]
     },
     {

notebooks/official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build an AutoML tabular regression workflow on Vertex AI Pipelines.\n",
         "\n",
-        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component)."
+        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component). Learn more about [Regression for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview)."
       ]
     },
     {

notebooks/official/pipelines/google_cloud_pipeline_components_dataproc_tabular.ipynb

@@ -74,7 +74,7 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this notebook, you learn how to build a Vertex AI pipeline and train a Random-forest model using Spark ML for loan-eligibility classification problem.  \n",
+        "In this notebook, you learn how to build a Vertex AI pipeline and train a random-forest model using Spark ML for loan-eligibility classification problem.  \n",
         "\n",
         "This tutorial uses the following Google Cloud ML services and resources:\n",
         "\n",
@@ -87,7 +87,7 @@
         "\n",
         "*   Use the `DataprocPySparkBatchOp` to preprocess data.\n",
         "*   Create a Vertex AI dataset resource on the training data.\n",
-        "*   Train a random forest model using Pyspark.\n",
+        "*   Train a random forest model using PySpark.\n",
         "*   Build a Vertex AI pipeline and run the training job.\n",
         "*   Use the Spark serving image in order to deploy a Spark model on Vertex AI Endpoint."
       ]
@@ -100,7 +100,7 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "The dataset is a preprocessed version of the [loan eligiability dataset](https://datasetsearch.research.google.com/search?src=2&query=Loan%20Eligible%20Dataset&docid=L2cvMTFsajJrM3EzcA%3D%3D)."
+        "The dataset is a preprocessed version of the [loan eligibility dataset](https://datasetsearch.research.google.com/search?src=2&query=Loan%20Eligible%20Dataset&docid=L2cvMTFsajJrM3EzcA%3D%3D)."
       ]
     },
     {
@@ -673,7 +673,7 @@
       "source": [
         "### Load preprocessing data\n",
         "\n",
-        "The notebook uses a preprocessed set of data you would read from the Vertex AI Feature Store. "
+        "The notebook uses a preprocessed set of data you read from the Vertex AI Feature Store. "
       ]
     },
     {
@@ -728,7 +728,7 @@
       "source": [
         "### Create the Docker repository\n",
         "\n",
-        "You create a Docker repository in the Artefact Registry for the custom dataproc image that you are going to create."
+        "You create a Docker repository in the Artifact Registry for the custom dataproc image that you are going to create."
       ]
     },
     {
@@ -739,7 +739,7 @@
       },
       "outputs": [],
       "source": [
-        "REPO_NAME = \"loan-eligiability-spark-demo\"\n",
+        "REPO_NAME = \"loan-eligibility-spark-demo\"\n",
         "\n",
         "!gcloud artifacts repositories create $REPO_NAME \\\n",
         "    --repository-format=docker \\\n",
@@ -771,8 +771,8 @@
         "\n",
         "from google.cloud import aiplatform as vertex_ai\n",
         "from kfp.v2 import compiler, dsl\n",
-        "from kfp.v2.dsl import (Artifact, ClassificationMetrics, Condition, Input,\n",
-        "                        Metrics, Output, component)"
+        "from kfp.v2.dsl import (ClassificationMetrics, Condition, Metrics, Output,\n",
+        "                        component)"
       ]
     },
     {
@@ -793,7 +793,7 @@
         "IMAGE_TAG = \"1.0.0\"\n",
         "\n",
         "# Pipeline\n",
-        "PIPELINE_NAME = \"pyspark-loan-eligiability-pipeline\"\n",
+        "PIPELINE_NAME = \"pyspark-loan-eligibility-pipeline\"\n",
         "PIPELINE_ROOT = f\"{BUCKET_URI}/pipelines\"\n",
         "PIPELINE_PACKAGE_PATH = str(BUILD_PATH / f\"pipeline_{UUID}.json\")\n",
         "RUNTIME_CONTAINER_IMAGE = f\"gcr.io/{PROJECT_ID}/{RUNTIME_IMAGE}:{IMAGE_TAG}\"\n",
@@ -815,10 +815,6 @@
         "    PROCESSED_DATA_URI,\n",
         "]\n",
         "\n",
-        "# Dataset\n",
-        "DATASET_NAME = f\"preprocessed-dataset-{UUID}\"\n",
-        "GCS_PREPROCESSED_URI = f\"{PROCESSED_DATA_URI}/*/?.csv\"\n",
-        "\n",
         "# Training\n",
         "TRAINING_PYTHON_FILE_URI = f\"{BUCKET_URI}/src/model_training.py\"\n",
         "MODEL_URI = f\"{BUCKET_URI}/deliverables/model/rfor/{UUID}/train_model\"\n",
@@ -863,6 +859,9 @@
         "    \"spark.jars.packages\": \"ml.combust.mleap:mleap-spark-base_2.12:0.20.0,ml.combust.mleap:mleap-spark_2.12:0.20.0\"\n",
         "}\n",
         "\n",
+        "# Experiment\n",
+        "EXPERIMENT_NAME = \"loan-eligibility\"\n",
+        "\n",
         "# Deploy\n",
         "SERVING_IMAGE_URI = f\"{REGION}-docker.pkg.dev/{PROJECT_ID}/{REPO_NAME}/spark-ml-serving\""
       ]
@@ -1122,11 +1121,11 @@
       "source": [
         "#### Create the source code for model-training\n",
         "\n",
-        "Create the `model_training.py` file for training a Random-forest classifier model on the training data. The training is performed using Spark ML inside a Spark session. The code fetches the training data from the Cloud storage bucket, processes it and trains the Random-forest model. The trained model and the metrics obtained from the trained model (like AUC-ROC, accuracy, precision etc.) are then saved to the provided output Cloud Storage path. This code accepts the following arguments:\n",
+        "Create the `model_training.py` file for training a random-forest classifier model on the training data. The training is performed using Spark ML inside a Spark session. The code fetches the training data from the Cloud storage bucket, processes it and trains the random-forest model. The trained model and the metrics obtained from the trained model (like AUC-ROC, accuracy, precision etc.) are then saved to the provided output Cloud Storage path. This code accepts the following arguments:\n",
         "\n",
-        "- `--train-path`: The GCS path of the training sample.\n",
-        "- `--model-path`: The GCS path to store the trained model.\n",
-        "- `--metrics-path`: The GCS path to store the metrics of model."
+        "- `--train-path`: The Cloud Storage path of the training sample.\n",
+        "- `--model-path`: The Cloud Storage path to store the trained model.\n",
+        "- `--metrics-path`: The Cloud Storage path to store the metrics of model."
       ]
     },
     {
@@ -1482,7 +1481,7 @@
       "source": [
         "#### Create the source code for hyperparameter-tuning\n",
         "\n",
-        "Create the `hp_tuning.py` file for tuning the hyperparameters of the Random-forest classifier model using crossvalidation. This code accepts the following arguments:\n",
+        "Create the `hp_tuning.py` file for tuning the hyperparameters of the random-forest classifier model using crossvalidation. This code accepts the following arguments:\n",
         "\n",
         "- `--train-path`: The GCS path of the training sample.\n",
         "- `--model-path`: The GCS path to store the trained model.\n",
@@ -1891,9 +1890,9 @@
       "source": [
         "### Build a custom Dataproc Serverless container image\n",
         "\n",
-        "Dataproc Serverless provides [default runtime images](https://cloud.google.com/dataproc-serverless/docs/concepts/versions/spark-runtime-versions). You can also use custom container images for your Dataproc Serverless workloads. \n",
+        "Dataproc Serverless provides default runtime images. Learn more about the [Dataproc Serverless Spark runtime releases](https://cloud.google.com/dataproc-serverless/docs/concepts/versions/spark-runtime-versions).\n",
         "\n",
-        "The steps in this section builds a custom container image that includes additional dependencies. The custom container image can be specified when using the `DataprocPySparkBatchOp` component to launch the workload within a pipeline."
+        "You can also use custom container images for your Dataproc Serverless workloads. The steps in this section builds a custom container image that includes additional dependencies. The custom container image can be specified when using the `DataprocPySparkBatchOp` component to launch the workload within a pipeline."
       ]
     },
     {
@@ -2010,7 +2009,7 @@
         "id": "ZXzI2xInqb3V"
       },
       "source": [
-        "#### Build the Dataproc Serverless custom runtime using Google Cloud Build\n",
+        "#### Build the Dataproc Serverless custom runtime using Cloud Build\n",
         "\n",
         "**Note:** this step may take approximately 5 to 10 minutes to complete."
       ]
@@ -2034,15 +2033,8 @@
       "source": [
         "### Build custom components for pipeline arguments\n",
         "\n",
-        "In order to pass job arguments, you create some custom components for each step of the pipeline. Next, you create a `register_model` component in order to register the PySpark model in Vertex AI Metadata. "
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "sSzUGhjq1SrR"
-      },
-      "source": [
+        "In order to pass job arguments, you create some custom components for each step of the pipeline.\n",
+        "\n",
         "#### Create component for passing args to preprocessing component\n",
         "\n",
         "The following component passes the args `--train-data-path` and `--out-process-path` in the required format for the preprocessing function defined earlier."
@@ -2086,9 +2078,7 @@
       "outputs": [],
       "source": [
         "@component(base_image=\"python:3.8-slim\")\n",
-        "def build_training_args(\n",
-        "    dataset_uri: Input[Artifact], train_path: str, model_path: str, metrics_path: str\n",
-        ") -> list:\n",
+        "def build_training_args(train_path: str, model_path: str, metrics_path: str) -> list:\n",
         "    return [\n",
         "        \"--train-path\",\n",
         "        train_path,\n",
@@ -2105,7 +2095,7 @@
         "id": "5VesQlnEm4qR"
       },
       "source": [
-        "#### Model Evaluation custom component\n",
+        "#### Create model evaluation custom component\n",
         "\n",
         "Define the component for processing the metrics for model evaluation. The `metrics_uri`, `metrics` and `plots` obtained as outputs from the model training component are further evaluated through this component."
       ]
@@ -2136,7 +2126,7 @@
         "\n",
         "    # Variables --------------------------------------------------------------------------------------------------------------------------\n",
         "    metrics_path = metrics_uri.replace(\"gs://\", \"/gcs/\")\n",
-        "    labels = [\"not eligiable\", \"eligiable\"]\n",
+        "    labels = [\"not eligible\", \"eligible\"]\n",
         "\n",
         "    # Helpers --------------------------------------------------------------------------------------------------------------------------\n",
         "    def calculate_roc(metrics, true, score):\n",
@@ -2207,7 +2197,6 @@
       "source": [
         "@component(base_image=\"python:3.8-slim\")\n",
         "def build_hpt_args(\n",
-        "    dataset_uri: Input[Artifact],\n",
         "    train_path: str,\n",
         "    model_path: str,\n",
         "    metrics_path: str,\n",
@@ -2258,7 +2247,7 @@
       "source": [
         "### Build the model serving container image\n",
         "\n",
-        "A *serving container image* is required to import your model into the Model Registry. The serving container image provides the model serving implementation for the model. The following replicates the instructions from [Serving Spark ML model using Vertex AI](https://cloud.google.com/architecture/spark-ml-model-with-vertexai) to build the serving container image.\n",
+        "A *serving container image* is required to import your model into the Model Registry. The serving container image provides the model serving implementation for the model. Learn more about [serving Spark ML models using Vertex AI](https://cloud.google.com/architecture/spark-ml-model-with-vertexai).\n",
         "\n",
         "**Note:** this step may take approximately 5 to 10 minutes to complete."
       ]
@@ -2333,7 +2322,9 @@
         "id": "0d83d9e80923"
       },
       "source": [
-        "The serving container requires the model schema in JSON format, which is read during container startup. See [Provide the model schema](https://cloud.google.com/architecture/spark-ml-model-with-vertexai#provide_the_model_schema) for more information.\n",
+        "### Define the schema for model serving\n",
+        "\n",
+        "The serving container requires the model schema in JSON format, which is read during container startup. Learn more about [providing the model schema](https://cloud.google.com/architecture/spark-ml-model-with-vertexai#provide_the_model_schema).\n",
         "\n",
         "Write the model schema file:"
       ]
@@ -2417,7 +2408,9 @@
         "id": "d0b88e26570a"
       },
       "source": [
-        "Copy the model schema configuration file to GCS. The serving container reads the model schema file location from the `AIP_STORAGE_URI` environment at startup. See [Import the model into Vertex AI](https://cloud.google.com/architecture/spark-ml-model-with-vertexai#import-the-model-into-vertex-ai) for more information."
+        "### Copy the model schema configuration file to GCS.\n",
+        "\n",
+        "The serving container reads the model schema file location from the `AIP_STORAGE_URI` environment at startup. See [Import the model into Vertex AI](https://cloud.google.com/architecture/spark-ml-model-with-vertexai#import-the-model-into-vertex-ai) for more information."
       ]
     },
     {
@@ -2437,9 +2430,9 @@
         "id": "wQd9M1_9bif7"
       },
       "source": [
-        "### Define your workflow using Kubeflow Pipelines\n",
+        "### Define your workflow as a Vertex AI Pipeline\n",
         "\n",
-        "Below, you use the Kubelflow Pipelines' DSL package to build your pipeline using the defined components and containers."
+        "Use the Kubeflow Pipelines SDK to define your workflow as a machine learning pipeline. The pipeline uses the custom components defined earlier, in addition to components from the `google-cloud-pipeline-components` package."
       ]
     },
     {
@@ -2455,8 +2448,6 @@
         "    preprocessing_main_python_file_uri: str = PREPROCESSING_PYTHON_FILE_URI,\n",
         "    train_data_path: str = FEATURES_TRAIN_URI,\n",
         "    preprocessed_data_path: str = PROCESSED_DATA_URI,\n",
-        "    dataset_name: str = DATASET_NAME,\n",
-        "    dataset_uri: str = GCS_PREPROCESSED_URI,\n",
         "    training_main_python_file_uri: str = TRAINING_PYTHON_FILE_URI,\n",
         "    train_path: str = PROCESSED_DATA_URI,\n",
         "    model_path: str = MODEL_URI,\n",
@@ -2477,8 +2468,6 @@
         "):\n",
         "    from google_cloud_pipeline_components.v1.dataproc import \\\n",
         "        DataprocPySparkBatchOp\n",
-        "    from google_cloud_pipeline_components.v1.dataset import \\\n",
-        "        TabularDatasetCreateOp\n",
         "    from google_cloud_pipeline_components.v1.endpoint import (EndpointCreateOp,\n",
         "                                                              ModelDeployOp)\n",
         "    from google_cloud_pipeline_components.v1.model import ModelUploadOp\n",
@@ -2498,21 +2487,12 @@
         "        subnetwork_uri=subnetwork_uri,\n",
         "    ).after(build_preprocessing_args_op)\n",
         "\n",
-        "    # create dataset\n",
-        "    create_dataset_op = TabularDatasetCreateOp(\n",
-        "        display_name=dataset_name,\n",
-        "        gcs_source=dataset_uri,\n",
-        "        project=project_id,\n",
-        "        location=location,\n",
-        "    ).after(data_preprocessing_op)\n",
-        "\n",
         "    # build training data args\n",
         "    build_training_args_op = build_training_args(\n",
-        "        dataset_uri=create_dataset_op.output,\n",
         "        train_path=train_path,\n",
         "        model_path=model_path,\n",
         "        metrics_path=metrics_path,\n",
-        "    ).after(create_dataset_op)\n",
+        "    ).after(data_preprocessing_op)\n",
         "\n",
         "    # training model\n",
         "    model_training_op = DataprocPySparkBatchOp(\n",
@@ -2535,7 +2515,6 @@
         "    ):\n",
         "\n",
         "        build_hpt_args_op = build_hpt_args(\n",
-        "            dataset_uri=create_dataset_op.output,\n",
         "            train_path=train_path,\n",
         "            model_path=hpt_model_path,\n",
         "            metrics_path=hpt_metrics_path,\n",
@@ -2620,7 +2599,9 @@
       "source": [
         "### Submit your pipeline run\n",
         "\n",
-        "Next, you use the Vertex AI Python SDK to submit and run your pipeline through Vertex AI Pipelines."
+        "Next, you use the Vertex AI Python SDK to submit and run your pipeline through Vertex AI Pipelines.\n",
+        "\n",
+        "The parameters, artifacts, and metrics produced from the pipeline run are automatically captured into Vertex AI Experiments as an experiment run."
       ]
     },
     {
@@ -2638,7 +2619,7 @@
         "    enable_caching=False,\n",
         ")\n",
         "\n",
-        "pipeline.submit(service_account=SERVICE_ACCOUNT)"
+        "pipeline.submit(service_account=SERVICE_ACCOUNT, experiment=EXPERIMENT_NAME)"
       ]
     },
     {
@@ -2663,6 +2644,33 @@
         "pipeline.wait()"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0574e2941aaf"
+      },
+      "source": [
+        "### (Optional) View experiment runs\n",
+        "\n",
+        "You can retrieve the parameters, artifacts, and metrics for all experiment runs as a pandas DataFrame. See [Compare and analyze runs](https://cloud.google.com/vertex-ai/docs/experiments/compare-analyze-runs) for more information on the topic."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4f5ab528a98e"
+      },
+      "outputs": [],
+      "source": [
+        "experiment_df = vertex_ai.get_experiment_df(experiment=EXPERIMENT_NAME)\n",
+        "\n",
+        "# Show successfully completed experiment runs, sorted by F1 score\n",
+        "experiment_df.query('state == \"COMPLETE\"').sort_values(\n",
+        "    \"metric.Test_f1-score\", ascending=False\n",
+        ")"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -2673,15 +2681,10 @@
         "\n",
         "You can request online predictions if the model was deployed to a Vertex AI endpoint. Use the `google-cloud-aiplatform` client library to request predictions, or you can use `curl`.\n",
         "\n",
-        "For this model, the prediction response contains the predicted label (`0 == not eligible`, `1 == eligible`) for each prediction instance that is sent to the endpoint."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "4c311f9fc363"
-      },
-      "source": [
+        "For this model, the prediction response contains the predicted label (`0 == not eligible`, `1 == eligible`) for each prediction instance that is sent to the endpoint.\n",
+        "\n",
+        "#### Use `google-cloud-aiplatform` to request online predictions\n",
+        "\n",
         "The following cell demonstrates how to use the `google-cloud-aiplatform` client library to request predictions from one or more instances."
       ]
     },
@@ -2708,6 +2711,8 @@
         "id": "1a2104c45e21"
       },
       "source": [
+        "#### Use `curl` to request online predictions\n",
+        "\n",
         "To use `curl`, first write the prediction instances to a file:"
       ]
     },
@@ -2786,12 +2791,7 @@
         "# Delete model\n",
         "model_list = vertex_ai.Model.list(filter=f'display_name=\"{MODEL_NAME}\"')\n",
         "for model in model_list:\n",
-        "    model.delete()\n",
-        "\n",
-        "# Delete dataset\n",
-        "dataset_list = vertex_ai.TabularDataset.list(filter=f'display_name=\"{DATASET_NAME}\"')\n",
-        "for dataset in dataset_list:\n",
-        "    dataset.delete()"
+        "    model.delete()"
       ]
     },
     {

notebooks/official/pipelines/pipelines_intro_kfp.ipynb

@@ -207,7 +207,7 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "PROJECT_ID = \"andy-1234-221921\"  # @param {type:\"string\"}\n",
         "\n",
         "# Set the project id\n",
         "! gcloud config set project {PROJECT_ID}"
@@ -590,8 +590,9 @@
       "outputs": [],
       "source": [
         "@component\n",
-        "def consumer(text1: str, text2: str, text3: str):\n",
-        "    print(f\"text1: {text1}; text2: {text2}; text3: {text3}\")"
+        "def consumer(text1: str, text2: str, text3: str) -> str:\n",
+        "    print(f\"text1: {text1}; text2: {text2}; text3: {text3}\")\n",
+        "    return f\"text1: {text1}; text2: {text2}; text3: {text3}\""
       ]
     },
     {

notebooks/official/pipelines/rapid_prototyping_bqml_automl.ipynb

@@ -33,19 +33,19 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/structured_data/rapid_prototyping_bqml_automl.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/rapid_prototyping_bqml_automl.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>  \n",
         "\n",
         "  <td>\n",
-        "<a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/structured_data/rapid_prototyping_bqml_automl.ipynb\" target='_blank'>\n",
+        "<a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/rapid_prototyping_bqml_automl.ipynb\" target='_blank'>\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "<a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/structured_data/rapid_prototyping_bqml_automl.ipynb\" target='_blank'>\n",
+        "<a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/pipelines/rapid_prototyping_bqml_automl.ipynb\" target='_blank'>\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -66,7 +66,7 @@
         "\n",
         "<img src=\"https://storage.googleapis.com/rafacarv-public-bucket-do-not-delete/abalone/automl_and_bqml.png\" />\n",
         "\n",
-        "Learn more about [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) and [BigQuery ML](https://cloud.google.com/bigquery-ml/docs/introduction)."
+        "Learn more about [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component) and [BigQuery ML components](https://cloud.google.com/vertex-ai/docs/pipelines/bigqueryml-component)."
       ]
     },
     {

notebooks/official/prediction/README.md

@@ -1,7 +1,4 @@
 
-### prediction 
-
-
 [Custom model batch prediction with feature filtering](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/prediction/custom_batch_prediction_feature_filter.ipynb)
 
 ```
@@ -15,3 +12,5 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Batch Prediction](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/get-batch-predictions).
+

notebooks/official/pytorch/README.md

@@ -1,7 +1,4 @@
 
-### pytorch 
-
-
 [Training, tuning and deploying a PyTorch text sentiment classification model on Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pytorch/pytorch-text-sentiment-classification-custom-train-deploy.ipynb)
 
 ```
@@ -19,3 +16,5 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+

notebooks/official/reduction_server/README.md

@@ -1,7 +1,4 @@
 
-### reduction_server 
-
-
 [PyTorch distributed training with Vertex AI Reduction Server](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/reduction_server/pytorch_distributed_training_reduction_server.ipynb)
 
 ```
@@ -16,3 +13,7 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+   Learn more about [Vertex AI Reduction Server](https://cloud.google.com/blog/topics/developers-practitioners/optimize-training-performance-reduction-server-vertex-ai).
+

notebooks/official/sdk/README.md

@@ -1,7 +1,4 @@
 
-### sdk 
-
-
 [AutoML Video Classification Example](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/sdk/SDK_AutoML_Video_Classification.ipynb)
 
 ```
@@ -19,6 +16,8 @@ The steps performed include the following:
 
 ```
 
+   Learn more about [AutoML Video](https://cloud.google.com/vertex-ai/docs/tutorials/video-classification-automl/training).
+
 
 [Custom training using Python package, managed text dataset, and TF Serving container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/sdk/SDK_Custom_Training_Python_Package_Managed_Text_Dataset_Tensorflow_Serving_Container.ipynb)
 
@@ -39,3 +38,5 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+

notebooks/official/structured_data/README.md

@@ -1,20 +0,0 @@
-
-### structured_data 
-
-
-[BQML and AutoML - Experimenting with Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/structured_data/rapid_prototyping_bqml_automl.ipynb)
-
-```
-Learn how to use `Vertex AI Predictions` for rapid prototyping a model.
-
-The steps performed include:
-
-- Creating a BigQuery and Vertex AI training dataset.
-- Training a BigQuery ML and AutoML model.
-- Extracting evaluation metrics from the BigQueryML and AutoML models.
-- Selecting the best trained model.
-- Deploying the best trained model.
-- Testing the deployed model infrastructure.
-
-```
-

notebooks/official/tabnet/README.md

@@ -1,7 +1,4 @@
 
-### tabnet 
-
-
 [Vertex AI Explainations with TabNet models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tabnet/ai-explanations-tabnet-algorithm.ipynb)
 
 ```
@@ -15,6 +12,8 @@ The steps performed are:
 
 ```
 
+   Learn more about [Tabular Workflow for TabNet](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/tabnet).
+
 
 [Vertex AI TabNet](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tabnet/tabnet_vertex_tutorial.ipynb)
 
@@ -31,3 +30,5 @@ The steps performed are:
 
 ```
 
+   Learn more about [Tabular Workflow for TabNet](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/tabnet).
+

notebooks/official/tabnet/ai-explanations-tabnet-algorithm.ipynb

@@ -65,7 +65,7 @@
         "\n",
         "The goal of the tutorial is to provide a sample plotting tool to visualize the output of TabNet, which is helpful in explaining the algorithm.\n",
         "\n",
-        "Learn more about [Vertex AI TabNet](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/tabnet) and [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview)."
+        "Learn more about [Tabular Workflow for TabNet](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/tabnet)."
       ]
     },
     {

notebooks/official/tabnet/tabnet_vertex_tutorial.ipynb

@@ -67,7 +67,7 @@
         "\n",
         "TabNet uses a machine learning technique called sequential attention to select which model features to reason from at each step in the model. This mechanism makes it possible to explain how the model arrives at its predictions and helps it learn more accurate models. Thanks to this design, TabNet not only outperforms other neural networks and decision trees but also provides interpretable feature attributions. Releasing TabNet as a First Party Trainer in Vertex AI means you'll be able to easily take advantage of TabNet's architecture and explainability and use it to train models on your own data. \n",
         "\n",
-        "Learn more about [Vertex AI TabNet](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/tabnet) and [Vertex AI Hyperparameter Tuning](https://cloud.google.com/vertex-ai/docs/training/hyperparameter-tuning-overview)."
+        "Learn more about [Tabular Workflow for TabNet](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/tabnet)."
       ]
     },
     {

notebooks/official/tabular_workflows/README.md

@@ -1,7 +1,4 @@
 
-### tabular_workflows 
-
-
 [TabNet Pipeline](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb)
 
 ```
@@ -14,6 +11,8 @@ The steps performed include:
 
 ```
 
+   Learn more about [Tabular Workflow for TabNet](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/tabnet).
+
 
 [Wide & Deep Pipeline](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tabular_workflows/wide_and_deep_on_vertex_pipelines.ipynb)
 
@@ -27,3 +26,5 @@ The steps performed include:
 
 ```
 
+   Learn more about [Tabular Workflow for Wide & Deep](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/wide-and-deep).
+

notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "This notebook showcases how to run the TabNet algorithm using Vertex AI Tabular Workflows.\n",
         "\n",
-        "Learn more about [Vertex AI TabNet](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/tabnet) and [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction)."
+        "Learn more about [Tabular Workflow for TabNet](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/tabnet)."
       ]
     },
     {

notebooks/official/tabular_workflows/wide_and_deep_on_vertex_pipelines.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "This notebook showcases how to run the Wide & Deep algorithm using Vertex AI Tabular Workflows.\n",
         "\n",
-        "Learn more about [Vertex AI Wide & Deep](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/wide-and-deep) and [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction)."
+        "Learn more about [Tabular Workflow for Wide & Deep](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/wide-and-deep)."
       ]
     },
     {

notebooks/official/tensorboard/README.md

@@ -1,5 +1,21 @@
 
-### tensorboard 
+[Vertex AI TensorBoard Custom Training with custom container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb)
+
+```
+Learn how to create a custom training job using custom containers, and monitor your training process on Vertex AI TensorBoard in near real time.
+
+The steps performed include:
+
+* Create docker repository & config.
+* Create a custom container image with your customized training code.
+* Setup service account and Google Cloud Storage buckets.
+* Create & launch your custom training job with your custom container.
+
+```
+
+   Learn more about [Vertex AI TensorBoard](https://cloud.google.com/vertex-ai/docs/experiments/tensorboard-overview).
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
 
 
 [Vertex AI TensorBoard custom training with prebuilt container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tensorboard/tensorboard_custom_training_with_prebuilt_container.ipynb)
@@ -16,6 +32,27 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI TensorBoard](https://cloud.google.com/vertex-ai/docs/experiments/tensorboard-overview).
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+
+[Profile model training performance using Vertex AI TensorBoard Profiler](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb)
+
+```
+Learn how to enable Vertex AI TensorBoard Profiler for custom training jobs.
+
+The steps performed include:
+
+- Setup a service account and a Cloud Storage bucket
+- Create a TensorBoard instance
+- Create and run a custom training job that enables TensorBoard Profiler
+- View the TensorBoard Profiler dashboard to debug your model training performance
+
+```
+
+   Learn more about [Vertex AI TensorBoard Profiler](https://cloud.google.com/vertex-ai/docs/experiments/tensorboard-profiler).
+
 
 [Vertex AI TensorBoard integration with Vertex AI Pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb)
 
@@ -30,33 +67,7 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI TensorBoard](https://cloud.google.com/vertex-ai/docs/experiments/tensorboard-overview).
 
-[Vertex AI TensorBoard Custom Training with Custom Container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb)
-
-```
-Learn how to create a custom training job using custom containers, and monitor your training process on Vertex AI TensorBoard in near real time.
-
-The steps performed include:
-
-* Create docker repository & config.
-* Create a custom container image with your customized training code.
-* Setup service account and Google Cloud Storage buckets.
-* Create & launch your custom training job with your custom container.
-
-```
-
-
-[Profile model training performance using Profiler](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb)
-
-```
-Learn how to enable Vertex AI TensorBoard Profiler for custom training jobs.
-
-The steps performed include:
-
-- Setup a service account and a Cloud Storage bucket
-- Create a TensorBoard instance
-- Create and run a custom training job that enables TensorBoard Profiler
-- View the TensorBoard Profiler dashboard to debug your model training performance
-
-```
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
 

notebooks/official/tensorboard/tensorboard_hyperparameter_tuning_with_hparams.ipynb

@@ -0,0 +1,794 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2022 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JAPoU8Sm5E6e"
+      },
+      "source": [
+        "# Vertex AI TensorBoard Hyperparameter Tuning with the HParams Dashboard\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tensorboard/tensorboard_hyperparameter_tuning_with_hparams.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tensorboard/tensorboard_hyperparameter_tuning_with_hparams.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/tensorboard/tensorboard_hyperparameter_tuning_with_hparams.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "24743cf4a1e1"
+      },
+      "source": [
+        "**_NOTE_**: This notebook has been tested in the following environments:\n",
+        "\n",
+        "* Python version = 3.8"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "### What is Vertex AI TensorBoard\n",
+        "\n",
+        "[Open source TensorBoard](https://www.tensorflow.org/tensorboard/get_started)\n",
+        "(TB) is a Google open source project for machine learning experiment\n",
+        "visualization. Vertex AI TensorBoard is an enterprise-ready managed\n",
+        "version of TensorBoard.\n",
+        "\n",
+        "Vertex AI TensorBoard provides various detailed visualizations, including the following:\n",
+        "\n",
+        "*   Tracking and visualizing metrics, such as loss and accuracy over time.\n",
+        "*   Visualizing model computational graphs (ops and layers).\n",
+        "*   Viewing histograms of weights, biases, or other tensors as they change over time.\n",
+        "*   Projecting embeddings to a lower dimensional space.\n",
+        "*   Displaying image, text, and audio samples.\n",
+        "\n",
+        "In addition to the powerful visualizations from\n",
+        "TensorBoard, Vertex AI TensorBoard provides the following benefits:\n",
+        "\n",
+        "*  A persistent, shareable link to your experiment's dashboard.\n",
+        "\n",
+        "*  A searchable list of all experiments in a project.\n",
+        "\n",
+        "*  Integrations with Vertex AI services for model training.\n",
+        "\n",
+        "*  Enterprise-grade security, privacy, and compliance.\n",
+        "\n",
+        "With Vertex AI TensorBoard, you can track, visualize, and compare\n",
+        "ML experiments and share them with your team.\n",
+        "\n",
+        "Learn more about [Vertex AI TensorBoard](https://cloud.google.com/vertex-ai/docs/experiments/tensorboard-overview)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d975e698c9a4"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "This tutorial shows you how to log hyperparameter experiment results in TensorFlow and visualize the results in TensorBoard's Hparams dashboard.\n",
+        "\n",
+        "This tutorial uses the following Vertex AI services and resources:\n",
+        "\n",
+        "- Vertex AI TensorBoard\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "* Adapt TensorFlow runs to log hyperparameters and metrics.\n",
+        "* Start runs and log them all under one parent directory.\n",
+        "* Visualize the results in TensorBoard's HParams dashboard."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "08d289fa873f"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "This tutorial uses the [FashionMNIST](https://github.com/zalandoresearch/fashion-mnist) dataset.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aed92deeb4a0"
+      },
+      "source": [
+        "### Costs\n",
+        "\n",
+        "This tutorial uses the following billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "\n",
+        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing),\n",
+        "and use the [Pricing Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1fD9UZaygyPG"
+      },
+      "source": [
+        "## Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or Vertex AI Workbench**, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "\n",
+        "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
+        "\n",
+        "- Git\n",
+        "- Python 3\n",
+        "- virtualenv\n",
+        "- Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "To quickly set up your environment to meet the requirements of this tutorial, perform the following:\n",
+        "\n",
+        "1. [Install and initialize the SDK](https://cloud.google.com/sdk/docs/).\n",
+        "\n",
+        "2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).\n",
+        "\n",
+        "3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3 and activate the virtual environment.\n",
+        "\n",
+        "4. Install Jupyter by running the following command in a terminal shell:\n",
+        "<br> `pip3 install jupyter`\n",
+        "\n",
+        "5. Launch Jupyter by running the following command in a terminal shell: <br> `jupyter notebook`\n",
+        "\n",
+        "6. Open this tutorial notebook in the Jupyter Notebook Dashboard."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "i7EUnXsZhAGF"
+      },
+      "source": [
+        "## Install dependencies\n",
+        "\n",
+        "Install the following packages required to run this tutorial notebook."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "th7tWguZiSN2"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform[tensorboard] tensorflow==2.7 {USER_FLAG} -q"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "58707a750154"
+      },
+      "source": [
+        "### Colab only: Uncomment the following cell to restart the kernel."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f200f10a1da3"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
+        "\n",
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "3. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
+        "\n",
+        "4. If you are running this notebook locally, install the [Cloud SDK](https://cloud.google.com/sdk)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oM1iC_MfAts1"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Set the region\n",
+        "\n",
+        "**Optional**: Update the 'REGION' variable to specify the region that you want to use. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nsN5NJKSu-GU"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sBCra4QMA2wR"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "To authenticate your Google Cloud account, follow the instructions for your Jupyter environment:"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "74ccc9e52986"
+      },
+      "source": [
+        "* **Vertex AI Workbench**\n",
+        "<br>You are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "de775a3773ba"
+      },
+      "source": [
+        "* **Local JupyterLab instance**\n",
+        "<br>Uncomment and run the following code:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "254614fa0c46"
+      },
+      "outputs": [],
+      "source": [
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ef21552ccea8"
+      },
+      "source": [
+        "* **Colab**\n",
+        "<br>Uncomment and run the following code:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "603adbbf0532"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "960505627ddf"
+      },
+      "source": [
+        "### Import libraries"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PyQmSRbKA8r-"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk,all"
+      },
+      "source": [
+        "### Initialize the Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "KllitKlIu-GW"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, location=REGION)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WjWD61gONRkw"
+      },
+      "source": [
+        "### Load TensorBoard and TensorFlow components\n",
+        "\n",
+        "Load the TensorBoard notebook extension and import TensorFlow and the TensorBoard HParams plugin.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "KSayPNqxfJC_"
+      },
+      "outputs": [],
+      "source": [
+        "# Load the TensorBoard notebook extension\n",
+        "%load_ext tensorboard\n",
+        "\n",
+        "# Clear any logs from previous runs\n",
+        "!rm -rf ./logs/\n",
+        "\n",
+        "# Import TensorFlow and the TensorBoard HParams plugin\n",
+        "import tensorflow as tf\n",
+        "from tensorboard.plugins.hparams import api as hp"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KJ4zE7rYfcvb"
+      },
+      "source": [
+        "### Download dataset\n",
+        "\n",
+        "Download the [FashionMNIST](https://github.com/zalandoresearch/fashion-mnist) dataset and scale it."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vHME9wnnfiMr"
+      },
+      "outputs": [],
+      "source": [
+        "fashion_mnist = tf.keras.datasets.fashion_mnist\n",
+        "\n",
+        "(x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()\n",
+        "x_train, x_test = x_train / 255.0, x_test / 255.0"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ofGSMru5r4kP"
+      },
+      "source": [
+        "## Set up the experiment\n",
+        "\n",
+        "Run an experiment by specifying values for the following hyperparameters:\n",
+        "\n",
+        "* Number of units in the first dense layer\n",
+        "* Dropout rate in the dropout layer\n",
+        "* Optimizer\n",
+        "\n",
+        "Specify the hyperparameter values for the experiment in TensorBoard.\n",
+        "\n",
+        "*Optional*: For more fine grained filtering of hyperparameters in the UI, provide domain information and specify which metrics should be displayed."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IG5sPLBAcDRy"
+      },
+      "outputs": [],
+      "source": [
+        "HP_NUM_UNITS = hp.HParam(\"num_units\", hp.Discrete([16, 32]))\n",
+        "HP_DROPOUT = hp.HParam(\"dropout\", hp.RealInterval(0.1, 0.2))\n",
+        "HP_OPTIMIZER = hp.HParam(\"optimizer\", hp.Discrete([\"adam\", \"sgd\"]))\n",
+        "\n",
+        "METRIC_ACCURACY = \"accuracy\"\n",
+        "\n",
+        "with tf.summary.create_file_writer(\"logs/hparam_tuning\").as_default():\n",
+        "    hp.hparams_config(\n",
+        "        hparams=[HP_NUM_UNITS, HP_DROPOUT, HP_OPTIMIZER],\n",
+        "        metrics=[hp.Metric(METRIC_ACCURACY, display_name=\"Accuracy\")],\n",
+        "    )"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cLNgBNA6srlk"
+      },
+      "source": [
+        "## Adapt TensorFlow runs to log hyperparameters and metrics\n",
+        "\n",
+        "The model will be quite simple: two dense layers with a dropout layer between them. The training code will look familiar, although the hyperparameters are no longer hardcoded. Instead, the hyperparameters are provided in an `hparams` dictionary and used throughout the training function:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "C-RSsrF4u-Fq"
+      },
+      "outputs": [],
+      "source": [
+        "def train_test_model(hparams):\n",
+        "    model = tf.keras.models.Sequential(\n",
+        "        [\n",
+        "            tf.keras.layers.Flatten(),\n",
+        "            tf.keras.layers.Dense(hparams[HP_NUM_UNITS], activation=tf.nn.relu),\n",
+        "            tf.keras.layers.Dropout(hparams[HP_DROPOUT]),\n",
+        "            tf.keras.layers.Dense(10, activation=tf.nn.softmax),\n",
+        "        ]\n",
+        "    )\n",
+        "    model.compile(\n",
+        "        optimizer=hparams[HP_OPTIMIZER],\n",
+        "        loss=\"sparse_categorical_crossentropy\",\n",
+        "        metrics=[\"accuracy\"],\n",
+        "    )\n",
+        "\n",
+        "    model.fit(\n",
+        "        x_train, y_train, epochs=1\n",
+        "    )  # Run with 1 epoch to speed things up for demo purposes\n",
+        "    _, accuracy = model.evaluate(x_test, y_test)\n",
+        "    return accuracy"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Esz3uqqCvLoK"
+      },
+      "source": [
+        "For each run, log an hparams summary with the hyperparameters and final accuracy:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HwR1PAv1vPER"
+      },
+      "outputs": [],
+      "source": [
+        "def run(run_dir, hparams):\n",
+        "    with tf.summary.create_file_writer(run_dir).as_default():\n",
+        "        hp.hparams(hparams)  # record the values used in this trial\n",
+        "        accuracy = train_test_model(hparams)\n",
+        "        tf.summary.scalar(METRIC_ACCURACY, accuracy, step=1)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0V_8soFFvU7b"
+      },
+      "source": [
+        "## Start runs and log them all under one parent directory\n",
+        "\n",
+        "You can now try multiple experiments, training each one with a different set of hyperparameters.\n",
+        "\n",
+        "For simplicity, use a grid search: try all combinations of the discrete parameters and just the lower and upper bounds of the real-valued parameter. For more complex scenarios, it might be more effective to choose each hyperparameter value randomly (this is called a random search). There are more advanced methods that can be used.\n",
+        "\n",
+        "Run a few experiments, which will take a few minutes:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6r2oO_PVvbdL"
+      },
+      "outputs": [],
+      "source": [
+        "session_num = 0\n",
+        "\n",
+        "for num_units in HP_NUM_UNITS.domain.values:\n",
+        "    for dropout_rate in (HP_DROPOUT.domain.min_value, HP_DROPOUT.domain.max_value):\n",
+        "        for optimizer in HP_OPTIMIZER.domain.values:\n",
+        "            hparams = {\n",
+        "                HP_NUM_UNITS: num_units,\n",
+        "                HP_DROPOUT: dropout_rate,\n",
+        "                HP_OPTIMIZER: optimizer,\n",
+        "            }\n",
+        "            run_name = \"run-%d\" % session_num\n",
+        "            print(\"--- Starting trial: %s\" % run_name)\n",
+        "            print({h.name: hparams[h] for h in hparams})\n",
+        "            run(\"logs/hparam_tuning/\" + run_name, hparams)\n",
+        "            session_num += 1"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6FJJwCclvslF"
+      },
+      "source": [
+        "## Visualize the results in Vertex AI TensorBoard's HParams tab"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BkbB5GEI3Ge3"
+      },
+      "source": [
+        "### Create Vertex AI Tensorboard\n",
+        "A Vertex AI TensorBoard instance, which is a regionalized resource storing your Vertex AI TensorBoard experiments, must be created before the experiments can be visualized. You can create multiple instances in a project. [documentation instructions](https://cloud.google.com/vertex-ai/docs/experiments/tensorboard-overview).\n",
+        "\n",
+        "Create a TensorBoard instance to be used by the training job."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lQ-d3j-I3ZWV"
+      },
+      "outputs": [],
+      "source": [
+        "TENSORBOARD_NAME = \"[your-tensorboard-name]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "if (\n",
+        "    TENSORBOARD_NAME == \"\"\n",
+        "    or TENSORBOARD_NAME is None\n",
+        "    or TENSORBOARD_NAME == \"[your-tensorboard-name]\"\n",
+        "):\n",
+        "    TENSORBOARD_NAME = PROJECT_ID + \"-tb-\"\n",
+        "\n",
+        "tensorboard = aiplatform.Tensorboard.create(\n",
+        "    display_name=TENSORBOARD_NAME, project=PROJECT_ID, location=REGION\n",
+        ")\n",
+        "TENSORBOARD_RESOURCE_NAME = tensorboard.gca_resource.name\n",
+        "print(\"TensorBoard resource name:\", TENSORBOARD_RESOURCE_NAME)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "27rERDqeJ2nE"
+      },
+      "source": [
+        "Set your TensorBoard Experiment name."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4OU4TMtFCn0_"
+      },
+      "outputs": [],
+      "source": [
+        "from datetime import datetime\n",
+        "\n",
+        "EXPERIMENT_NAME = \"[your-experiment-run-name]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "if (\n",
+        "    EXPERIMENT_NAME == \"\"\n",
+        "    or EXPERIMENT_NAME is None\n",
+        "    or EXPERIMENT_NAME == \"[your-experiment-run-name]\"\n",
+        "):\n",
+        "    EXPERIMENT_NAME = \"experiment\" + datetime.now().strftime(\"%H-%M-%S\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f1D2oU3K8Ys0"
+      },
+      "source": [
+        "Upload the log to your Vertex AI TensorBoard"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TyXFVQuRv0-X"
+      },
+      "outputs": [],
+      "source": [
+        "!tb-gcp-uploader --one_shot=True --tensorboard_resource_name=$TENSORBOARD_RESOURCE_NAME --logdir=\"logs/hparam_tuning/\" --experiment_name=$EXPERIMENT_NAME"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OFe3qRyh9Wjl"
+      },
+      "source": [
+        "Click the generated TensorBoard link and click on \"HParams\" at the top.\n",
+        "\n",
+        "The left pane of the dashboard provides filtering capabilities that are active across all the views in the HParams dashboard:\n",
+        "\n",
+        "- Filter which hyperparameters/metrics are shown in the dashboard\n",
+        "- Filter which hyperparameter/metrics values are shown in the dashboard\n",
+        "- Filter on run status (running, success, ...)\n",
+        "- Sort by hyperparameter/metric in the table view\n",
+        "- Number of session groups to show (useful for performance when there are many experiments)\n",
+        "\n",
+        "The HParams dashboard has three different views, with various useful information:\n",
+        "\n",
+        "* The **Table View** lists the runs, their hyperparameters, and their metrics.\n",
+        "* The **Parallel Coordinates View** shows each run as a line going through an axis for each hyperparemeter and metric. Click and drag the mouse on any axis to mark a region which will highlight only the runs that pass through it. This can be useful for identifying which groups of hyperparameters are most important. The axes themselves can be re-ordered by dragging them.\n",
+        "* The **Scatter Plot View** shows plots comparing each hyperparameter/metric with each metric. This can help identify correlations. Click and drag to select a region in a specific plot and highlight those sessions across the other plots.\n",
+        "\n",
+        "A table row, a parallel coordinates line, and a scatter plot market can be clicked to see a plot of the metrics as a function of training steps for that session (although in this tutorial only one step is used for each run)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TpV-iwP9qw9c"
+      },
+      "source": [
+        "## Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "sx_vKniMq9ZX"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# Delete endpoint resource\n",
+        "# e.g. `endpoint.delete()`\n",
+        "\n",
+        "# Delete model resource\n",
+        "# e.g. `model.delete()`\n",
+        "\n",
+        "# Delete Cloud Storage objects that were created\n",
+        "delete_bucket = False\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
+        "    ! gsutil -m rm -r $BUCKET_URI"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "tensorboard_hyperparameter_tuning_with_hparams.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/official/tensorboard/tensorboard_profiler_custom_training_with_prebuilt_container.ipynb

@@ -0,0 +1,869 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2022 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "l2mMvIUG9meX"
+      },
+      "source": [
+        "# Profile model training performance using Vertex AI TensorBoard Profiler in custom training with prebuilt container\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tensorboard/tensorboard_profiler_custom_training_with_prebuilt_container.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tensorboard/tensorboard_profiler_custom_training_with_prebuilt_container.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/tensorboard/tensorboard_profiler_custom_training_with_prebuilt_container.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "The TensorFlow Profiler is a powerful tool that can help you to diagnose and debug performance bottlenecks, and make your model train faster. This tutorial demonstrates how to enable the TensorBoard Profiler in Vertex AI for custom training with a prebuilt container.\n",
+        "\n",
+        "Learn more about [Vertex AI TensorBoard Profiler](https://cloud.google.com/vertex-ai/docs/experiments/tensorboard-profiler)."
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dmfmQL6w84pS"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn how to enable the TensorBoard Profiler in Vertex AI for custom training jobs with a prebuilt container.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud AI services:\n",
+        "\n",
+        "- Vertex AI Training\n",
+        "- Vertex AI TensorBoard\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Prepare your custom training code and load your training code as a Python package to a prebuilt container\n",
+        "- Create and run a custom training job that enables the TensorBoard Profiler\n",
+        "- View the TensorBoard Profiler dashboard to debug your model training performance\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zfXf0r-K81Y-"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [mnist dataset](https://www.tensorflow.org/datasets/catalog/mnist) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview).\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "I3KFLvpq87rs"
+      },
+      "source": [
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* Cloud Storage\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
+        "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ze4-nDLfK4pw"
+      },
+      "source": [
+        "## Installation\n",
+        "\n",
+        "Install the following packages required to execute this notebook."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2b4ef9b72d43"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install --upgrade --quiet google-cloud-aiplatform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aUw6ibN-n5Za"
+      },
+      "source": [
+        "### Colab only: Uncomment the following cell to restart the kernel."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "FM12wbWhn7w0"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
+        "\n",
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LgFWLeJfoGQu"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "3. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
+        "\n",
+        "4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8ckyxpX_oSzD"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zY8DKBoVoVy3"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mSQjVQmMosMl"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Se9FWWhLotvB"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IfJRIMBpo5Pg"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "acFN0s3So9-Y"
+      },
+      "source": [
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dQ_mNwuapE5T"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cR_MzpknpGgM"
+      },
+      "outputs": [],
+      "source": [
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "h-MuVI_ypJfw"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "BeaQlCwMpQUT"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3ivZkPUjpaFz"
+      },
+      "source": [
+        "**4. Setup service account and permissions**\n",
+        "\n",
+        "A service account will be used to create custom training jobs. If you do not want to use your project's Compute Engine service account, set SERVICE_ACCOUNT to another service account ID. You can create a service account by following the [instructions](https://cloud.google.com/iam/docs/creating-managing-service-accounts#creating)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vYE3b942wza4"
+      },
+      "outputs": [],
+      "source": [
+        "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WWIxsCJFCg5Z"
+      },
+      "outputs": [],
+      "source": [
+        "# Grant Cloud Storage permission.\n",
+        "! gcloud projects add-iam-policy-binding $PROJECT_ID \\\n",
+        "        --member=\"serviceAccount:$SERVICE_ACCOUNT\" \\\n",
+        "        --role=\"roles/storage.admin\" \\\n",
+        "        --quiet\n",
+        "\n",
+        "# Grant AI Platform permission.\n",
+        "! gcloud projects add-iam-policy-binding $PROJECT_ID \\\n",
+        "        --member=\"serviceAccount:$SERVICE_ACCOUNT\" \\\n",
+        "        --role=\"roles/aiplatform.user\" \\\n",
+        "        --quiet\n",
+        "\n",
+        "! gcloud projects get-iam-policy $PROJECT_ID \\\n",
+        "        --filter=bindings.members:serviceAccount:$SERVICE_ACCOUNT"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OKtKGmr9pfr6"
+      },
+      "source": [
+        "### Create a Cloud Storage bucket\n",
+        "\n",
+        "Create a storage bucket to store intermediate artifacts such as datasets."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "In3aQanwYjFB"
+      },
+      "outputs": [],
+      "source": [
+        "BUCKET_URI = \"gs://your-bucket-name-unique\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GOaOsIjxp0oB"
+      },
+      "source": [
+        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Wn5QiIl2p16e"
+      },
+      "outputs": [],
+      "source": [
+        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ankcS-vtp7Wv"
+      },
+      "source": [
+        "### Import libraries"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WffSImMvp-Po"
+      },
+      "outputs": [],
+      "source": [
+        "from google.cloud import aiplatform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OMrAJ8RGqBQu"
+      },
+      "source": [
+        "### Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "AWRzBFExqERG"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-ayTbNdi62_t"
+      },
+      "source": [
+        "### Create a TensorBoard instance\n",
+        "\n",
+        "A Vertex AI TensorBoard instance, which is a regionalized resource storing your Vertex AI TensorBoard experiments, must be created before the experiments can be visualized. You can create multiple instances in a project. You can use command  `gcloud ai tensorboards list` to get a list of your existing TensorBoard instances."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9c3QrDTZdaxk"
+      },
+      "source": [
+        "#### Set your TensorBoard instance display name\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "azlwb__AX8gs"
+      },
+      "outputs": [],
+      "source": [
+        "TENSORBOARD_NAME = \"your-tensorboard-unique\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vJrWKK0mY7H7"
+      },
+      "source": [
+        "#### Create a TensorBoard instance\n",
+        "\n",
+        "If you don't have a TensorBoard instance, create one by running the following cell:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JqVNsRFrc_78"
+      },
+      "outputs": [],
+      "source": [
+        "tensorboard = aiplatform.Tensorboard.create(\n",
+        "    display_name=TENSORBOARD_NAME, project=PROJECT_ID, location=REGION\n",
+        ")\n",
+        "\n",
+        "TENSORBOARD_INSTANCE_NAME = tensorboard.resource_name\n",
+        "print(\"TensorBoard instance name:\", TENSORBOARD_INSTANCE_NAME)"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yoR29gW2S24w"
+      },
+      "source": [
+        "## Train a model\n",
+        "\n",
+        "To train a model using your custom training code, choose one of the following options:\n",
+        "\n",
+        "- **Prebuilt container**: Load your custom training code as a Python package to a prebuilt container image from Google Cloud.\n",
+        "\n",
+        "- **Custom container**: Create your own container image that contains your custom training code.\n",
+        "\n",
+        "In this tutorial, you will train a custom model using a prebuilt container."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "syw3GabNGgJz"
+      },
+      "source": [
+        "### Examine the training package\n",
+        "\n",
+        "#### Package layout\n",
+        "\n",
+        "Before you start the training, let's take a look at how a Python package is assembled for a custom training job. When extracted, the package contains the following:\n",
+        "\n",
+        "- PKG-INFO\n",
+        "- README.md\n",
+        "- setup.cfg\n",
+        "- setup.py\n",
+        "- trainer\n",
+        "  - \\_\\_init\\_\\_.py\n",
+        "  - task.py\n",
+        "\n",
+        "The files `setup.cfg` and `setup.py` are the instructions for installing the package into the operating environment of the docker image."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b58ZAbysGkRo"
+      },
+      "outputs": [],
+      "source": [
+        "PYTHON_PACKAGE_APPLICATION_DIR = \"app\"\n",
+        "\n",
+        "source_package_file_name = f\"{PYTHON_PACKAGE_APPLICATION_DIR}/dist/trainer-0.1.tar.gz\"\n",
+        "python_package_gcs_uri = f\"{BUCKET_URI}/trainer-0.1.tar.gz\"\n",
+        "\n",
+        "# Make folder for Python training script\n",
+        "! rm -rf {PYTHON_PACKAGE_APPLICATION_DIR}\n",
+        "! mkdir {PYTHON_PACKAGE_APPLICATION_DIR}\n",
+        "\n",
+        "# Add package information\n",
+        "! touch {PYTHON_PACKAGE_APPLICATION_DIR}/README.md\n",
+        "\n",
+        "# Make the training subfolder\n",
+        "! mkdir {PYTHON_PACKAGE_APPLICATION_DIR}/trainer\n",
+        "! touch {PYTHON_PACKAGE_APPLICATION_DIR}/trainer/__init__.py"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lj7hIeAXGrzg"
+      },
+      "outputs": [],
+      "source": [
+        "%%writefile ./{PYTHON_PACKAGE_APPLICATION_DIR}/setup.py\n",
+        "\n",
+        "from setuptools import find_packages\n",
+        "from setuptools import setup\n",
+        "import setuptools\n",
+        "\n",
+        "from distutils.command.build import build as _build\n",
+        "import subprocess\n",
+        "\n",
+        "REQUIRED_PACKAGES = [\n",
+        "    'google-cloud-aiplatform[cloud_profiler]>=1.20.0',\n",
+        "]\n",
+        "\n",
+        "setup(\n",
+        "    install_requires=REQUIRED_PACKAGES,\n",
+        "    packages=find_packages(),\n",
+        "    include_package_data=True,\n",
+        "    name='trainer',\n",
+        "    version='0.1',\n",
+        "    url=\"wwww.google.com\",\n",
+        "    description='Vertex AI | Training | Python Package'\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hyAwgsoQmaYI"
+      },
+      "source": [
+        "#### Prepare the training script\n",
+        "\n",
+        "The file `trainer/task.py` is the Python script for executing the custom training job.\n",
+        "\n",
+        "Your training code must be configured to write TensorBoard logs to a Cloud Storage bucket, the location of which Vertex AI Training automatically makes available through a predefined environment variable, `AIP_TENSORBOARD_LOG_DIR`. This can usually be done by providing `os.environ['AIP_TENSORBOARD_LOG_DIR']` as the log directory to the open source TensorBoard log writing APIs. For example, in TensorFlow 2.x, you can use following code to create a `tensorboard_callback`:\n",
+        "\n",
+        "    tensorboard_callback = tf.keras.callbacks.TensorBoard(\n",
+        "      log_dir=os.environ['AIP_TENSORBOARD_LOG_DIR'],\n",
+        "      histogram_freq=1)\n",
+        "`AIP_TENSORBOARD_LOG_DIR` is in the `BASE_OUTPUT_DIR` that you provide when creating the custom training job.\n",
+        "\n",
+        "To enable Vertex AI TensorBoard Profiler for your training job, add the following to your training script:\n",
+        "\n",
+        "Add the cloud_profiler import at your top level imports:\n",
+        "\n",
+        "    from google.cloud.aiplatform.training_utils import cloud_profiler\n",
+        "\n",
+        "\n",
+        "Initialize the cloud_profiler plugin by adding:\n",
+        "\n",
+        "\n",
+        "    cloud_profiler.init()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8JCgWW7Au1w8"
+      },
+      "outputs": [],
+      "source": [
+        "%%writefile ./{PYTHON_PACKAGE_APPLICATION_DIR}/trainer/task.py\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "import argparse\n",
+        "import os\n",
+        "import sys, traceback\n",
+        "from google.cloud.aiplatform.training_utils import cloud_profiler\n",
+        "\n",
+        "\"\"\"Train an mnist model and use cloud_profiler for profiling.\"\"\"\n",
+        "\n",
+        "def _create_model():\n",
+        "    model = tf.keras.models.Sequential(\n",
+        "        [\n",
+        "            tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
+        "            tf.keras.layers.Dense(128, activation=\"relu\"),\n",
+        "            tf.keras.layers.Dropout(0.2),\n",
+        "            tf.keras.layers.Dense(10),\n",
+        "        ]\n",
+        "    )\n",
+        "    return model\n",
+        "\n",
+        "\n",
+        "def main(args):\n",
+        "    print('Initialize the profiler ...')\n",
+        "    cloud_profiler.init()\n",
+        "    print('The profiler initiated.')\n",
+        "\n",
+        "    print('Loading and preprocessing data ...')\n",
+        "    mnist = tf.keras.datasets.mnist\n",
+        "\n",
+        "    (x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
+        "    x_train, x_test = x_train / 255.0, x_test / 255.0\n",
+        "\n",
+        "    print('Creating and training model ...')\n",
+        "\n",
+        "    model = _create_model()\n",
+        "    model.compile(\n",
+        "      optimizer=\"adam\",\n",
+        "      loss=tf.keras.losses.sparse_categorical_crossentropy,\n",
+        "      metrics=[\"accuracy\"],\n",
+        "    )\n",
+        "\n",
+        "    log_dir = \"logs\"\n",
+        "    if 'AIP_TENSORBOARD_LOG_DIR' in os.environ:\n",
+        "      log_dir = os.environ['AIP_TENSORBOARD_LOG_DIR']\n",
+        "\n",
+        "    print('Setting up the TensorBoard callback ...')\n",
+        "    tensorboard_callback = tf.keras.callbacks.TensorBoard(\n",
+        "        log_dir=log_dir,\n",
+        "        histogram_freq=1)\n",
+        "\n",
+        "    print('Training model ...')\n",
+        "    model.fit(\n",
+        "        x_train,\n",
+        "        y_train,\n",
+        "        epochs=args.epochs,\n",
+        "        verbose=0,\n",
+        "        callbacks=[tensorboard_callback],\n",
+        "    )\n",
+        "    print('Training completed.')\n",
+        "\n",
+        "    print('Saving model ...')\n",
+        "\n",
+        "    model_dir = \"model\"\n",
+        "    if 'AIP_MODEL_DIR' in os.environ:\n",
+        "      model_dir = os.environ['AIP_MODEL_DIR']\n",
+        "    tf.saved_model.save(model, model_dir)\n",
+        "\n",
+        "    print('Model saved at ' + model_dir)\n",
+        "\n",
+        "\n",
+        "if __name__ == \"__main__\":\n",
+        "    parser = argparse.ArgumentParser()\n",
+        "    parser.add_argument(\n",
+        "        \"--epochs\", type=int, default=100, help=\"Number of epochs to run model.\"\n",
+        "    )\n",
+        "\n",
+        "    args = parser.parse_args()\n",
+        "    main(args)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ihYFahRAr6sj"
+      },
+      "source": [
+        "#### Create a source distribution\n",
+        "\n",
+        "You create a source distribution with your training application and upload the source distribution to your Cloud Storage bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-XhccshCHQeb"
+      },
+      "outputs": [],
+      "source": [
+        "!cd {PYTHON_PACKAGE_APPLICATION_DIR} && python3 setup.py sdist --formats=gztar\n",
+        "\n",
+        "!gsutil cp {source_package_file_name} {python_package_gcs_uri}\n",
+        "\n",
+        "!gsutil ls -l {python_package_gcs_uri}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "k4e6OYmimqTR"
+      },
+      "source": [
+        "### Create and run the custom training job\n",
+        "\n",
+        "Configure a [custom job](https://cloud.google.com/vertex-ai/docs/training/create-custom-job) with the [pre-built container](https://cloud.google.com/vertex-ai/docs/training/pre-built-containers) image for training code packaged as Python source distribution."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "t8GeVXjWHxuZ"
+      },
+      "outputs": [],
+      "source": [
+        "JOB_NAME = \"tensorboard-job-unique\"\n",
+        "MACHINE_TYPE = \"n1-standard-4\"\n",
+        "TRAIN_IMAGE = \"us-docker.pkg.dev/vertex-ai/training/tf-cpu.2-9:latest\"\n",
+        "base_output_dir = f\"{BUCKET_URI}/{JOB_NAME}\"\n",
+        "python_module_name = \"trainer.task\"\n",
+        "\n",
+        "EPOCHS = 20\n",
+        "training_args = [\n",
+        "    \"--epochs=\" + str(EPOCHS),\n",
+        "]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "B3JC7T3bH9Vy"
+      },
+      "outputs": [],
+      "source": [
+        "job = aiplatform.CustomPythonPackageTrainingJob(\n",
+        "    display_name=JOB_NAME,\n",
+        "    python_package_gcs_uri=python_package_gcs_uri,\n",
+        "    python_module_name=python_module_name,\n",
+        "    container_uri=TRAIN_IMAGE,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "51hKGTbU32Eg"
+      },
+      "source": [
+        "#### Run the custom training job\n",
+        "\n",
+        "Next, you run the custom job to start the training job by invoking the method `run`.\n",
+        "\n",
+        "**NOTE:** When using Vertex AI SDK for Python for submitting a training job, it creates a [training pipeline](https://cloud.google.com/vertex-ai/docs/training/create-training-pipeline) which launches the custom job on Vertex AI Training service."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oIyfos1rIAx2"
+      },
+      "outputs": [],
+      "source": [
+        "job.run(\n",
+        "    replica_count=1,\n",
+        "    machine_type=MACHINE_TYPE,\n",
+        "    base_output_dir=base_output_dir,\n",
+        "    tensorboard=TENSORBOARD_INSTANCE_NAME,\n",
+        "    service_account=SERVICE_ACCOUNT,\n",
+        "    args=training_args,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JkEe2Nb_85UD"
+      },
+      "source": [
+        "## View the TensorBoard Profiler dashboard\n",
+        "\n",
+        "When the custom job state switches to `Running`, you can access the Vertex AI TensorBoard Profiler dashboard through the Custom jobs page or the Experiments page on the Google Cloud console.\n",
+        "\n",
+        "The Google Cloud guide to [Profile model training performance using Profiler](https://cloud.google.com/vertex-ai/docs/experiments/tensorboard-profiler) provides detailed instructions for accessing the Vertex AI TensorBoard Profiler dashboard and capturing a profiling session.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TpV-iwP9qw9c"
+      },
+      "source": [
+        "## Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WR-ZhQ9XwpRI"
+      },
+      "outputs": [],
+      "source": [
+        "delete_bucket = False\n",
+        "\n",
+        "job.delete()\n",
+        "tensorboard.delete()\n",
+        "\n",
+        "if delete_bucket and \"BUCKET_URI\" in globals():\n",
+        "    ! gsutil -m rm -r $BUCKET_URI"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "tensorboard_profiler_custom_training_with_prebuilt_container.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/official/training/README.md

@@ -1,5 +1,20 @@
 
-### training 
+[Get started with Vertex AI Distributed Training](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/training/get_started_with_vertex_distributed_training.ipynb)
+
+```
+Learn how to use `Vertex AI Distributed Training` for when training with `Vertex AI`.
+
+The steps performed include:
+
+- `MirroredStrategy`: Train on a single VM with multiple GPUs.
+- `MultiWorkerMirroredStrategy`: Train on multiple VMs with automatic setup of replicas.
+- `MultiWorkerMirroredStrategy`: Train on multiple VMs with fine grain control of replicas.
+- `ReductionServer`: Train on multiple VMS and sync updates across VMS with `Vertex AI Reduction Server`.
+- `TPUTraining`: Train with multiple Cloud TPUs.
+
+```
+
+   Learn more about [Vertex AI Distributed Training](https://cloud.google.com/vertex-ai/docs/training/distributed-training).
 
 
 [Run hyperparameter tuning for a TensorFlow model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/training/hyperparameter_tuning_tensorflow.ipynb)
@@ -15,6 +30,8 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Hyperparameter Tuning](https://cloud.google.com/vertex-ai/docs/training/hyperparameter-tuning-overview).
+
 
 [PyTorch image classification multi-node distributed data parallel training on cpu using Vertex training with custom container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/training/multi_node_ddp_gloo_vertex_training_with_custom_container.ipynb)
 
@@ -31,6 +48,8 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
 
 [PyTorch Image Classification Multi-Node Distributed Data Parallel Training on GPU using Vertex AI Training with Custom Container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/training/multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb)
 
@@ -47,6 +66,8 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
 
 [Create a distributed custom training job](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/training/xgboost_data_parallel_training_on_cpu_using_dask.ipynb)
 
@@ -62,3 +83,5 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+

notebooks/official/training/xgboost_data_parallel_training_on_cpu_using_dask.ipynb

@@ -61,7 +61,7 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This tutorial shows you how to create a distributed custom training job on Vertex AI that can handle large amounts of training data. \n",
+        "This tutorial shows you how to create a distributed custom training job on Vertex AI that can handle large amounts of training data.\n",
         "\n",
         "Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
       ]
@@ -78,8 +78,7 @@
         "\n",
         "This tutorial uses the following Google Cloud ML services:\n",
         "\n",
-        "- `Vertex AI SDK`\n",
-        "- `CustomContainerTrainingJob`\n",
+        "- `Vertex AI Training`\n",
         "- `Artifact Registry`\n",
         "\n",
         "The steps performed include:\n",
@@ -98,7 +97,7 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "This tutorial uses the <a href=\"https://scikit-learn.org/stable/auto_examples/datasets/plot_iris_dataset.html\">IRIS dataset</a>, which consists of different types of irises.\n"
+        "This tutorial uses the <a href=\"https://scikit-learn.org/stable/auto_examples/datasets/plot_iris_dataset.html\">IRIS dataset</a>, which predicts the iris species.\n"
       ]
     },
     {
@@ -108,7 +107,7 @@
       },
       "source": [
         "### Costs\n",
-        " \n",
+        "\n",
         "This tutorial uses billable components of Google Cloud:\n",
         "\n",
         "* Vertex AI\n",
@@ -477,7 +476,7 @@
         "id": "Xx_z9JQlrNwG"
       },
       "source": [
-        "# Create a custom training Python package \n",
+        "# Create a custom training Python package\n",
         "\n",
         "Before you can perform local training, you must a create a training script file and a docker file.\n",
         "\n",
@@ -492,17 +491,7 @@
       },
       "outputs": [],
       "source": [
-        "PYTHON_PACKAGE_APPLICATION_DIR = \"trainer\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "yjeHKqHwr4rV"
-      },
-      "outputs": [],
-      "source": [
+        "PYTHON_PACKAGE_APPLICATION_DIR = \"trainer\"\n",
         "!mkdir -p $PYTHON_PACKAGE_APPLICATION_DIR"
       ]
     },
@@ -574,14 +563,28 @@
         "    \"\"\"\n",
         "    return subprocess.check_call(cmd, stdout=sys.stdout, stderr=sys.stderr, shell=True)\n",
         "\n",
+        "\n",
         "def get_chief_ip(cluster_config_dict):\n",
-        "    ip_address = cluster_config_dict['cluster']['workerpool0'][0].split(\":\")[0]\n",
+        "    if 'workerpool0' in cluster_config_dict['cluster']:\n",
+        "      ip_address = cluster_config_dict['cluster']['workerpool0'][0].split(\":\")[0]\n",
+        "    else:\n",
+        "      # if the job is not distributed, 'chief' will be populated instead of\n",
+        "      # workerpool0.\n",
+        "      ip_address = cluster_config_dict['cluster']['chief'][0].split(\":\")[0]\n",
+        "\n",
         "    print('The ip address of workerpool 0 is : {}'.format(ip_address))\n",
         "    return ip_address\n",
         "\n",
         "def get_chief_port(cluster_config_dict):\n",
-        "    print(\"The open port is: {}\".format(cluster_config_dict['open_ports'][0]))\n",
-        "    return cluster_config_dict['open_ports'][0]\n",
+        "\n",
+        "    if \"open_ports\" in cluster_config_dict:\n",
+        "      port = cluster_config_dict['open_ports'][0]\n",
+        "    else:\n",
+        "      # Use any port for the non-distributed job.\n",
+        "      port = 7777\n",
+        "    print(\"The open port is: {}\".format(port))\n",
+        "\n",
+        "    return port\n",
         "\n",
         "if __name__ == '__main__':\n",
         "    cluster_config_str = os.environ.get('CLUSTER_SPEC')\n",
@@ -599,7 +602,7 @@
         "      proc_scheduler = launch('dask-scheduler --dashboard --dashboard-address 8888 --port {} &'.format(chief_port))\n",
         "      print('Done the dask scheduler.', flush=True)\n",
         "\n",
-        "      client = Client(chief_address)\n",
+        "      client = Client(chief_address, timeout=1200)\n",
         "      print('Waiting the scheduler to be connected.', flush=True)\n",
         "      client.wait_for_workers(1)\n",
         "\n",
@@ -610,7 +613,7 @@
         "      wait(X)\n",
         "      wait(y)\n",
         "      dtrain = DaskDMatrix(client, X, y)\n",
-        "      \n",
+        "\n",
         "      output = xgb.dask.train(client, XGB_PARAMS, dtrain,  num_boost_round=100, evals=[(dtrain, 'train')])\n",
         "      print(\"Output: {}\".format(output), flush=True)\n",
         "      print(\"Saving file to: {}\".format(MODEL_FILE), flush=True)\n",
@@ -623,6 +626,8 @@
         "      blob.upload_from_filename(MODEL_FILE)\n",
         "      print(\"Saved file to: {}/{}\".format(MODEL_DIR, MODEL_FILE), flush=True)\n",
         "\n",
+        "      # Waiting 10 mins to connect the Dask dashboard\n",
+        "      time.sleep(60 * 10)\n",
         "      client.shutdown()\n",
         "\n",
         "    else:\n",
@@ -630,7 +635,10 @@
         "      client = Client(chief_address, timeout=1200)\n",
         "      print('client: {}.'.format(client), flush=True)\n",
         "      launch('dask-worker {}'.format(chief_address))\n",
-        "      print('Done with the dask worker.', flush=True)\n"
+        "      print('Done with the dask worker.', flush=True)\n",
+        "\n",
+        "      # Waiting 10 mins to connect the Dask dashboard\n",
+        "      time.sleep(60 * 10)\n"
       ]
     },
     {
@@ -639,7 +647,8 @@
         "id": "MxsT4Vaos2W5"
       },
       "source": [
-        "### Write the docker file"
+        "### Write the docker file\n",
+        "The docker file is used to build the custom training container and passed to the Vertex Training."
       ]
     },
     {
@@ -654,14 +663,20 @@
         "FROM us-docker.pkg.dev/vertex-ai/training/tf-cpu.2-9:latest\n",
         "WORKDIR /root\n",
         "\n",
+        "# Update the keyring in order to run apt-get update.\n",
+        "RUN rm -rf /usr/share/keyrings/cloud.google.gpg\n",
+        "RUN rm -rf /etc/apt/sources.list.d/google-cloud-sdk.list\n",
+        "RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -\n",
+        "RUN echo \"deb https://packages.cloud.google.com/apt cloud-sdk main\" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list\n",
+        "\n",
         "RUN apt-get update\n",
         "RUN apt-get install -y telnet netcat iputils-ping  net-tools\n",
-        "RUN python3.8 -m pip install dask==2022.7.1 distributed==2022.7.1 bokeh==2.1.1 dask-cuda  --upgrade\n",
-        "RUN python3.8 -m pip install 'xgboost>=1.4.2' 'dask-ml[complete]==2022.5.27'  #'dask[complete]==2022.7,1' --upgrade\n",
+        "RUN python3.8 -m pip install 'xgboost>=1.4.2' 'dask-ml[complete]==2022.5.27' 'dask[complete]==2022.7.1' --upgrade\n",
+        "RUN python3.8 -m pip install dask==2022.7.1 distributed==2022.7.1 bokeh==2.4.3 dask-cuda==22.8.0  --upgrade\n",
         "RUN python3.8 -m pip install gcsfs --upgrade\n",
         "\n",
         "\n",
-        "## Make sure gsutil will use the default service account\n",
+        "# Make sure gsutil will use the default service account\n",
         "RUN echo '[GoogleCompute]\\nservice_account = default' > /etc/boto.cfg\n",
         "\n",
         "# Copies the trainer code\n",
@@ -736,10 +751,10 @@
       },
       "outputs": [],
       "source": [
-        "DEPLOY_IMAGE = (\n",
+        "TRAIN_IMAGE = (\n",
         "    f\"{REGION}-docker.pkg.dev/\" + PROJECT_ID + f\"/{PRIVATE_REPO}\" + \"/dask_support\"\n",
         ")\n",
-        "print(\"Deployment:\", DEPLOY_IMAGE)"
+        "print(\"Deployment:\", TRAIN_IMAGE)"
       ]
     },
     {
@@ -788,8 +803,8 @@
       "outputs": [],
       "source": [
         "if not IS_COLAB:\n",
-        "    ! docker build -t $DEPLOY_IMAGE -f Dockerfile .\n",
-        "    ! docker push $DEPLOY_IMAGE"
+        "    ! docker build -t $TRAIN_IMAGE -f Dockerfile .\n",
+        "    ! docker push $TRAIN_IMAGE"
       ]
     },
     {
@@ -812,7 +827,7 @@
       "outputs": [],
       "source": [
         "if IS_COLAB:\n",
-        "    !  gcloud builds submit --timeout=1800s --region={REGION} --tag $DEPLOY_IMAGE"
+        "    !  gcloud builds submit --timeout=1800s --region={REGION} --tag $TRAIN_IMAGE"
       ]
     },
     {
@@ -860,11 +875,12 @@
         "replica_count = 2\n",
         "machine_type = \"n1-standard-4\"\n",
         "display_name = \"test_display_name\"\n",
+        "DEPLOY_IMAGE = \"us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-8:latest\"\n",
         "\n",
         "custom_container_training_job = aiplatform.CustomContainerTrainingJob(\n",
         "    display_name=display_name,\n",
-        "    model_serving_container_image_uri=\"us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-8:latest\",\n",
-        "    container_uri=DEPLOY_IMAGE,\n",
+        "    model_serving_container_image_uri=DEPLOY_IMAGE,\n",
+        "    container_uri=TRAIN_IMAGE,\n",
         ")\n",
         "\n",
         "custom_container_training_job.run(\n",
@@ -886,6 +902,157 @@
         "print(f\"GCS Output URI Prefix: {gcs_output_uri_prefix}\")"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tVktIbToRpmR"
+      },
+      "source": [
+        "### Access the Dask dashboard"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uVvxLj8GRsM6"
+      },
+      "source": [
+        "You can also create a training job with gcloud command.  With gcloud command, you can specify the field enableWebAccess and enableDashboardAccess. enableWebAccess enables the interactive shell for the job and enableDashboardAccess allows the dask dashboard to be accessed."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pkOQtyDsRwsS"
+      },
+      "outputs": [],
+      "source": [
+        "%%bash -s \"$BUCKET_URI/output\" \"$TRAIN_IMAGE\"\n",
+        "\n",
+        "cat <<EOF >config.yaml\n",
+        "enableDashboardAccess: true\n",
+        "enableWebAccess: true\n",
+        "# Creates two worker pool. The first worker pool is a chief and the second is\n",
+        "# a worker.\n",
+        "workerPoolSpecs:\n",
+        "  - machineSpec:\n",
+        "      machineType: n1-standard-8\n",
+        "    replicaCount: 1\n",
+        "    containerSpec:\n",
+        "      imageUri: $2\n",
+        "  - machineSpec:\n",
+        "      machineType: n1-standard-8\n",
+        "    replicaCount: 1\n",
+        "    containerSpec:\n",
+        "      imageUri: $2\n",
+        "baseOutputDirectory:\n",
+        "  outputUriPrefix: $1\n",
+        "EOF\n",
+        "cat config.yaml"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d5FLoTWzSNw7"
+      },
+      "source": [
+        "The following command creates a training job."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1MPj-NnpSQ1U"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud ai custom-jobs create --region=us-central1 --config=config.yaml --display-name={display_name}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "onb40Ge0SVKh"
+      },
+      "source": [
+        "Once the job is created. You can use the output `gcloud ai custom-jobs describe` command to print the field webAccessUris. The interactive shell has the key with the format \"workerpool0-0\", while the dashboard uri has the key with the format \"workerpool0-0:\" + port number. Note: You have to access the links while the job is running."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FqFwDvWCSYFX"
+      },
+      "source": [
+        "#### Troubleshooting"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MjbPElukSiZt"
+      },
+      "source": [
+        "The [interactive shell](https://cloud.google.com/vertex-ai/docs/training/monitor-debug-interactive-shell) can be used to debugging the access of the dask dashboard. You can get the dashboard point by the following command."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "z9eNOtYUTzJW"
+      },
+      "outputs": [],
+      "source": [
+        "# Note the following command should run inside the interactive shell.\n",
+        "# printenv | grep AIP_DASHBOARD_PORT"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2SCRLCkpUDNM"
+      },
+      "source": [
+        "Then you can check if there are dashboard monitoring the port."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fK74qU78ULPS"
+      },
+      "outputs": [],
+      "source": [
+        "# Note the following command should run inside the interactive shell.\n",
+        "# netstat -ntlp"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8ucKHMGFUUF4"
+      },
+      "source": [
+        "You can manually turn up the dashboard instance."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gJrjiePvUip0"
+      },
+      "outputs": [],
+      "source": [
+        "# Note the following command should run inside the interactive shell.\n",
+        "# dask-scheduler --dashboard-address :port_number"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -919,7 +1086,8 @@
         "\n",
         "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "- Cloud Storage Bucket"
+        "- Cloud Storage Bucket\n",
+        "- Cloud Vertex Training Job"
       ]
     },
     {
@@ -936,13 +1104,14 @@
         "! gsutil rm -rf $gcs_output_uri_prefix\n",
         "\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
+        "    ! gsutil rm -r $BUCKET_URI\n",
+        "\n",
+        "custom_container_training_job.delete()"
       ]
     }
   ],
   "metadata": {
     "colab": {
-      "collapsed_sections": [],
       "name": "xgboost_data_parallel_training_on_cpu_using_dask.ipynb",
       "toc_visible": true
     },

notebooks/official/vizier/README.md

@@ -1,7 +1,4 @@
 
-### vizier 
-
-
 [Optimizing multiple objectives with Vertex AI Vizier](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb)
 
 ```
@@ -11,3 +8,5 @@ Learn how to use `Vertex AI Vizier` to optimize a multi-objective study.
 
 ```
 
+   Learn more about [Vertex AI Vizier](https://cloud.google.com/vertex-ai/docs/vizier/overview).
+

notebooks/official/workbench/README.md

@@ -1,38 +1,47 @@
 
-### sentiment_analysis 
-
-
-[Sentiment Analysis using AutoML Natural Language and Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/sentiment_analysis/Sentiment_Analysis.ipynb)
+[Train a multi-class classification model for ads-targeting](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb)
 
 ```
-<a name="section-2"></a>
-
-
-
-```
-
-
-### exploratory_data_analysis 
-
-
-[Interactive exploratory analysis of BigQuery data in a notebook](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/exploratory_data_analysis/explore_data_in_bigquery_with_workbench.ipynb)
-
-```
-Learn about various ways to explore and gain insights from BigQuery data in a Jupyter notebook environment.
+Learn how to collect data from BigQuery, preprocess it, and train a multi-class classification model on an e-commerce dataset.
 
 The steps performed include:
 
-- Using Python & SQL to query public data in BigQuery
-- Exploring the dataset using BigQuery INFORMATION_SCHEMA
-- Creating interactive elements to help explore interesting parts of the data
-- Doing some exploratory correlation and time series analysis
-- Creating static and interactive outputs (data tables and plots) in the notebook
-- Saving some outputs to Cloud Storage
+- Fetch the required data from BigQuery
+- Preprocess the data
+- Train a TensorFlow (>=2.4) classification model
+- Evaluate the loss for the trained model
+- Automate the notebook execution using the executor feature
+- Save the model to a Cloud Storage path
+- Clean up the created resources
 
 ```
 
+&nbsp;&nbsp;&nbsp;Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction).
 
-### demand_forecasting 
+&nbsp;&nbsp;&nbsp;Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+
+[Taxi fare prediction using the Chicago Taxi Trips dataset](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/chicago_taxi_fare_prediction/chicago_taxi_fare_prediction.ipynb)
+
+```
+The goal of this notebook is to provide an overview on the latest Vertex AI features like **Explainable AI** and **BigQuery in Notebooks** by trying to solve a taxi fare prediction problem.
+
+The steps performed include:
+
+- Loading the dataset using "BigQuery in Notebooks".
+- Performing exploratory data analysis on the dataset.
+- Feature selection and preprocessing.
+- Building a linear regression model using scikit-learn.
+- Configuring the model for Vertex Explainable AI.
+- Deploying the model to Vertex AI.
+- Testing the deployed model.
+- Clean up.
+
+```
+
+&nbsp;&nbsp;&nbsp;Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction).
+
+&nbsp;&nbsp;&nbsp;Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview).
 
 
 [Forecasting retail demand with Vertex AI and BigQuery ML](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/demand_forecasting/forecasting-retail-demand.ipynb)
@@ -51,21 +60,216 @@ The steps performed include:
 
 ```
 
+&nbsp;&nbsp;&nbsp;Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction).
 
-### predictive_maintainance 
+&nbsp;&nbsp;&nbsp;Learn more about [BigQuery ML](https://cloud.google.com/bigquery-ml/docs/managing-models-vertex).
+
+
+[Interactive exploratory analysis of BigQuery data in a notebook](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/exploratory_data_analysis/explore_data_in_bigquery_with_workbench.ipynb)
+
+```
+Learn about various ways to explore and gain insights from BigQuery data in a Jupyter notebook environment.
+
+The steps performed include:
+
+- Using Python & SQL to query public data in BigQuery
+- Exploring the dataset using BigQuery INFORMATION_SCHEMA
+- Creating interactive elements to help explore interesting parts of the data
+- Doing some exploratory correlation and time series analysis
+- Creating static and interactive outputs (data tables and plots) in the notebook
+- Saving some outputs to Cloud Storage
+
+```
+
+&nbsp;&nbsp;&nbsp;Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction).
+
+&nbsp;&nbsp;&nbsp;Learn more about [BigQuery ML](https://cloud.google.com/vertex-ai/docs/beginner/bqml).
+
+
+[Build a fraud detection model on Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/fraud_detection/fraud-detection-model.ipynb)
+
+```
+This tutorial demonstrates data analysis and model-building using a synthetic financial dataset.
+
+The steps performed include:
+
+- Installation of required libraries
+- Reading the dataset from a Cloud Storage bucket
+- Performing exploratory analysis on the dataset
+- Preprocessing the dataset
+- Training a random forest model using scikit-learn
+- Saving the model to a Cloud Storage bucket
+- Creating a Vertex AI model resource and deploying to an endpoint
+- Running the What-If Tool on test data
+- Un-deploying the model and cleaning up the model resources
+
+```
+
+&nbsp;&nbsp;&nbsp;Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction).
+
+&nbsp;&nbsp;&nbsp;Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+
+[Churn prediction for game developers using Google Analytics 4 and BigQuery ML](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/gaming_churn_prediction/churn_prediction_for_game_developers.ipynb)
+
+```
+Learn how to train, evaluate a propensity model in BigQuery ML.
+
+The steps performed include:
+
+* Explore an export of Google Analytics 4 data on BigQuery.
+* Prepare the training data using demographic, behavioral data, and labels (churn/not-churn).
+* Train an XGBoost model using BigQuery ML.
+* Evaluate a model using BigQuery ML.
+* Make predictions on which users will churn using BigQuery ML.
+
+```
+
+&nbsp;&nbsp;&nbsp;Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction).
+
+&nbsp;&nbsp;&nbsp;Learn more about [BigQuery ML](https://cloud.google.com/vertex-ai/docs/beginner/bqml).
+
+
+[Inventory prediction on ecommerce data using Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/inventory-prediction/inventory_prediction.ipynb)
+
+```
+This tutorial shows you how to do exploratory data analysis, preprocess data, train model, evaluate model, deploy model, configure What-If Tool.
+
+The steps performed include:
+
+* Load the dataset from BigQuery using the "BigQuery in Notebooks" integration.
+* Analyze the dataset.
+* Preprocess the features in the dataset.
+* Build a random forest classifier model that predicts whether a product will get sold in the next 60 days.
+* Evaluate the model.
+* Deploy the model using Vertex AI.
+* Configure and test with the What-If Tool.
+
+```
+
+&nbsp;&nbsp;&nbsp;Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction).
+
+&nbsp;&nbsp;&nbsp;Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
 
 
 [Predictive Maintenance using Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb)
 
 ```
-<a name="section-2"></a>
+Learn how to the executor feature of Vertex AI Workbench to automate a workflow to train and deploy a model.
 
+The steps performed are:
 
+- Loading the required dataset from a Cloud Storage bucket.
+- Analyzing the fields present in the dataset.
+- Selecting the required data for the predictive maintenance model.
+- Training an XGBoost regression model for predicting the remaining useful life.
+- Evaluating the model.
+- Running the notebook end-to-end as a training job using Executor.
+- Deploying the model on Vertex AI.
+- Clean up.
 
 ```
 
+&nbsp;&nbsp;&nbsp;Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction).
 
-### subscriber_churn_prediction 
+&nbsp;&nbsp;&nbsp;Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+
+[Analysis of pricing optimization on CDM Pricing Data](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/pricing_optimization/pricing-optimization.ipynb)
+
+```
+The objective of this notebook is to build a pricing optimization model using BigQuery ML.
+
+The steps performed include:
+
+- Load the required dataset from a Cloud Storage bucket.
+- Analyze the fields present in the dataset.
+- Process the data to build a model.
+- Build a BigQuery ML forecast model on the processed data.
+- Get forecasted values from the BigQuery ML model.
+- Interpret the forecasts to identify the best prices.
+- Clean up.
+
+```
+
+&nbsp;&nbsp;&nbsp;Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction).
+
+&nbsp;&nbsp;&nbsp;Learn more about [BigQuery ML](https://cloud.google.com/vertex-ai/docs/beginner/bqml).
+
+
+[Sentiment Analysis using AutoML Natural Language and Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/sentiment_analysis/Sentiment_Analysis.ipynb)
+
+```
+Learn how to train and deploy an AutoML sentiment analysis model, and make predictions.
+
+The steps performed are:
+
+- Loading the required data. 
+- Preprocessing the data.
+- Selecting the required data for the model.
+- Loading the dataset into Vertex AI managed datasets.
+- Training a sentiment model using AutoML Text training.
+- Evaluating the model.
+- Deploying the model on Vertex AI.
+- Getting predictions.
+- Clean up.
+
+```
+
+&nbsp;&nbsp;&nbsp;Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction).
+
+&nbsp;&nbsp;&nbsp;Learn more about [AutoML Text](https://cloud.google.com/vertex-ai/docs/tutorials/text-classification-automl/training).
+
+
+[Digest and analyze data from BigQuery with Dataproc](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/spark/spark_bigquery.ipynb)
+
+```
+This notebook tutorial runs an Apache Spark job that fetches data from the BigQuery "GitHub Activity Data" dataset, queries the data, and then writes the results back to BigQuery.
+
+The steps performed are:
+
+- Setting up a Google Cloud project and Dataproc cluster.
+- Configuring the spark-bigquery-connector.
+- Ingesting data from BigQuery into a Spark DataFrame.
+- Preprocessing ingested data.
+- Querying the most frequently used programming language in monoglot repos.
+- Querying the average size (MB) of code in each language stored in monoglot repos.
+- Querying the languages files most frequently found together in polyglot repos.
+- Writing the query results back into BigQuery.
+- Deleting the resources created for this notebook tutorial.
+
+```
+
+&nbsp;&nbsp;&nbsp;Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction).
+
+&nbsp;&nbsp;&nbsp;Learn more about [Dataproc](https://cloud.google.com/vertex-ai/docs/pipelines/dataproc-component).
+
+
+[SparkML with Dataproc and BigQuery](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/spark/spark_ml.ipynb)
+
+```
+This tutorial runs an Apache SparkML job that fetches data from the BigQuery dataset, performs exploratory data analysis, cleans the data, executes feature engineering, trains the model, evaluates the model, outputs results, and saves the model to a Cloud Storage bucket.
+
+The steps performed are:
+
+- Sets up a Google Cloud project and Dataproc cluster.
+- Creates a Cloud Storage bucket and a BigQuery dataset.
+- Configures the spark-bigquery-connector.
+- Ingests BigQuery data into a Spark DataFrame.
+- Performa Exploratory Data Analysis (EDA).
+- Visualizes the data with samples.
+- Cleans the data.
+- Selects features.
+- Trains the model.
+- Outputs results.
+- Saves the model to a Cloud Storage bucket.
+- Deletes the resources created for the tutorial.
+
+```
+
+&nbsp;&nbsp;&nbsp;Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction).
+
+&nbsp;&nbsp;&nbsp;Learn more about [Dataproc](https://cloud.google.com/vertex-ai/docs/pipelines/dataproc-component).
 
 
 [Telecom subscriber churn prediction on Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb)
@@ -88,113 +292,7 @@ The steps performed include:
 
 ```
 
+&nbsp;&nbsp;&nbsp;Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction).
 
-### spark 
-
-
-[SparkML with Dataproc and BigQuery](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/spark/spark_ml.ipynb)
-
-```
-This tutorial runs an Apache SparkML job that fetches data from the BigQuery dataset, performs exploratory data analysis, cleans the data, executes feature engineering, trains the model, evaluates the model, outputs results, and saves the model to a Cloud Storage bucket.
-
-
-
-```
-
-
-[Digest and analyze data from BigQuery with Dataproc](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/spark/spark_bigquery.ipynb)
-
-```
-This notebook tutorial runs an Apache Spark job that fetches data from the BigQuery "GitHub Activity Data" dataset, queries the data, and then writes the results back to BigQuery.
-
-
-
-```
-
-
-### ads_targetting 
-
-
-[Train a multi-class classification model for ads-targeting](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb)
-
-```
-Learn how to collect data from BigQuery, preprocess it, and train a multi-class classification model on an e-commerce dataset.
-
-The steps performed include:
-
-- Fetch the required data from BigQuery
-- Preprocess the data
-- Train a TensorFlow (>=2.4) classification model
-- Evaluate the loss for the trained model
-- Automate the notebook execution using the executor feature
-- Save the model to a Cloud Storage path
-- Clean up the created resources
-
-```
-
-
-### inventory-prediction 
-
-
-[Inventory prediction on ecommerce data using Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/inventory-prediction/inventory_prediction.ipynb)
-
-```
-This tutorial shows you how to do exploratory data analysis, preprocess data, train model, evaluate model, deploy model, configure What-If Tool.
-
-The steps performed include:
-
-* Load the dataset from BigQuery using the "BigQuery in Notebooks" integration.
-* Analyze the dataset.
-* Preprocess the features in the dataset.
-* Build a random forest classifier model that predicts whether a product will get sold in the next 60 days.
-* Evaluate the model.
-* Deploy the model using Vertex AI.
-* Configure and test with the What-If Tool.
-
-```
-
-
-### fraud_detection 
-
-
-[Build a fraud detection model on Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/fraud_detection/fraud-detection-model.ipynb)
-
-```
-This tutorial demonstrates data analysis and model-building using a synthetic financial dataset.
-
-The steps performed include:
-
-- Installation of required libraries
-- Reading the dataset from a Cloud Storage bucket
-- Performing exploratory analysis on the dataset
-- Preprocessing the dataset
-- Training a random forest model using scikit-learn
-- Saving the model to a Cloud Storage bucket
-- Creating a Vertex AI model resource and deploying to an endpoint
-- Running the What-If Tool on test data
-- Un-deploying the model and cleaning up the model resources
-
-```
-
-
-### chicago_taxi_fare_prediction 
-
-
-[Taxi fare prediction using the Chicago Taxi Trips dataset](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/chicago_taxi_fare_prediction/chicago_taxi_fare_prediction.ipynb)
-
-```
-The goal of this notebook is to provide an overview on the latest Vertex AI features like **Explainable AI** and **BigQuery in Notebooks** by trying to solve a taxi fare prediction problem.
-
-The steps performed include:
-
-- Loading the dataset using "BigQuery in Notebooks".
-- Performing exploratory data analysis on the dataset.
-- Feature selection and preprocessing.
-- Building a linear regression model using scikit-learn.
-- Configuring the model for Vertex Explainable AI.
-- Deploying the model to Vertex AI.
-- Testing the deployed model.
-- Clean up.
-
-```
+&nbsp;&nbsp;&nbsp;Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview).
 

notebooks/official/workbench/exploratory_data_analysis/explore_data_in_bigquery_with_workbench.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "This notebook is written for data analysts and data scientists who have data in BigQuery and want to perform exploratory data analysis to gather insights from that data in an interactive environment.\n",
         "\n",
-        "Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction) and [BigQuery](https://cloud.google.com/bigquery)."
+        "Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction) and Learn more about [BigQuery ML](https://cloud.google.com/vertex-ai/docs/beginner/bqml)."
       ]
     },
     {

notebooks/official/workbench/gaming_churn_prediction/churn_prediction_for_game_developers.ipynb

@@ -92,7 +92,7 @@
         "\n",
         "This tutorial shows you how to train, evaluate a propensity model in BigQuery ML to predict user retention on a mobile game, based on app measurement data from Google Analytics 4.\n",
         "\n",
-        "Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction) and [BigQuery ML](https://cloud.google.com/bigquery-ml/docs/managing-models-vertex)."
+        "Learn more about [Vertex AI Workbench](https://cloud.google.com/vertex-ai/docs/workbench/introduction) and Learn more about [BigQuery ML](https://cloud.google.com/vertex-ai/docs/beginner/bqml)."
       ]
     },
     {